Start fleshing out new extension API.

6 files changed, 939 insertions, 54 deletions

diff --git a/ChangeLog b/ChangeLog
index 1d89d639..2bee6d06 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2012-05-15         Arnold D. Robbins     <arnold@skeeve.com>
+
+	* awk.h: Include "gawkapi.h" to get IOBUF.
+	* gawkapi.h: Considerable updates.
+	* gawkapi.c: New file. Start at implementing the APIs.
+
 2012-05-13         Andrew J. Schorr     <aschorr@telemetry-investments.com>
 
 	* TODO.xgawk: Update to reflect recent discussions and deletion of
diff --git a/awk.h b/awk.h
index f79df717..31ed1d38 100644
--- a/awk.h
+++ b/awk.h
@@ -875,7 +875,9 @@ typedef struct exp_instruction {
 /* Op_store_var */
 #define initval         x.xn
 
-
+#if 1
+#include "gawkapi.h"
+#else
 typedef struct iobuf {
 	const char *name;       /* filename */
 	int fd;                 /* file descriptor */
@@ -909,6 +911,7 @@ typedef struct iobuf {
 #		define  IOP_CLOSED      8
 #		define  IOP_AT_START    16
 } IOBUF;
+#endif
 
 typedef void (*Func_ptr)(void);
 
diff --git a/extension/ChangeLog b/extension/ChangeLog
index a3526863..8292be08 100644
--- a/extension/ChangeLog
+++ b/extension/ChangeLog
@@ -1,3 +1,10 @@
+2012-05-15         Arnold D. Robbins     <arnold@skeeve.com>
+
+	* filefuncs2.c: New file implementing chdir and stat using the
+	new interface.
+
+	Everything else is temporarily broken.
+
 2012-05-13         Andrew J. Schorr     <aschorr@telemetry-investments.com>
 
 	* filefuncs.c (array_set): Add a comment discussing the use of unref
diff --git a/extension/filefuncs2.c b/extension/filefuncs2.c
new file mode 100644
index 00000000..79fae616
--- /dev/null
+++ b/extension/filefuncs2.c
@@ -0,0 +1,383 @@
+/*
+ * filefuncs.c - Builtin functions that provide initial minimal iterface
+ *		 to the file system.
+ *
+ * Arnold Robbins, update for 3.1, Mon Nov 23 12:53:39 EST 1998
+ * Arnold Robbins and John Haque, update for 3.1.4, applied Mon Jun 14 13:55:30 IDT 2004
+ */
+
+/*
+ * Copyright (C) 2001, 2004, 2005, 2010, 2011 the Free Software Foundation, Inc.
+ * 
+ * This file is part of GAWK, the GNU implementation of the
+ * AWK Programming Language.
+ * 
+ * GAWK is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * GAWK is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "gawkapi.h"
+
+static gawk_api_t *api;		/* for convenience macros to work */
+static awk_ext_id_t *ext_id;
+
+int plugin_is_GPL_compatible;
+
+/*  do_chdir --- provide dynamically loaded chdir() builtin for gawk */
+
+int
+do_chdir(int nargs)
+{
+	const awk_value_t *newdir;
+	awk_value_t result;
+	int ret = -1;
+
+	if (do_lint && nargs != 1)
+		lintwarn(ext_id, "chdir: called with incorrect number of arguments");
+
+	newdir = get_curfunc_param(ext_id, 0, AWK_PARAM_STRING);
+	ret = chdir(newdir->str_value.str);
+	if (ret < 0)
+		update_ERRNO_int(ext_id, errno);
+
+	result.val_type = AWK_NUMBER;
+	result.num_value = ret;
+	set_return_value(ext_id, & result);
+	return 1;
+}
+
+/* format_mode --- turn a stat mode field into something readable */
+
+static char *
+format_mode(unsigned long fmode)
+{
+	static char outbuf[12];
+	static struct mode_map {
+		int mask;
+		int rep;
+	} map[] = {
+		{ S_IRUSR, 'r' }, { S_IWUSR, 'w' }, { S_IXUSR, 'x' },
+		{ S_IRGRP, 'r' }, { S_IWGRP, 'w' }, { S_IXGRP, 'x' },
+		{ S_IROTH, 'r' }, { S_IWOTH, 'w' }, { S_IXOTH, 'x' },
+	};
+	int i, j, k;
+
+	strcpy(outbuf, "----------");
+	/* first, get the file type */
+	i = 0;
+	switch (fmode & S_IFMT) {
+#ifdef S_IFSOCK
+	case S_IFSOCK:
+		outbuf[i] = 's';
+		break;
+#endif
+#ifdef S_IFLNK
+	case S_IFLNK:
+		outbuf[i] = 'l';
+		break;
+#endif
+	case S_IFREG:
+		outbuf[i] = '-';	/* redundant */
+		break;
+	case S_IFBLK:
+		outbuf[i] = 'b';
+		break;
+	case S_IFDIR:
+		outbuf[i] = 'd';
+		break;
+#ifdef S_IFDOOR	/* Solaris weirdness */
+	case S_IFDOOR:
+		outbuf[i] = 'D';
+		break;
+#endif /* S_IFDOOR */
+	case S_IFCHR:
+		outbuf[i] = 'c';
+		break;
+#ifdef S_IFIFO
+	case S_IFIFO:
+		outbuf[i] = 'p';
+		break;
+#endif
+	}
+
+	for (j = 0, k = sizeof(map)/sizeof(map[0]); j < k; j++) {
+		i++;
+		if ((fmode & map[j].mask) != 0)
+			outbuf[i] = map[j].rep;
+	}
+
+	i++;
+	outbuf[i] = '\0';
+
+	/* setuid bit */
+	if ((fmode & S_ISUID) != 0) {
+		if (outbuf[3] == 'x')
+			outbuf[3] = 's';
+		else
+			outbuf[3] = 'S';
+	}
+
+	/* setgid without execute == locking */
+	if ((fmode & S_ISGID) != 0) {
+		if (outbuf[6] == 'x')
+			outbuf[6] = 's';
+		else
+			outbuf[6] = 'l';
+	}
+
+	/* the so-called "sticky" bit */
+	if ((fmode & S_ISVTX) != 0) {
+		if (outbuf[9] == 'x')
+			outbuf[9] = 't';
+		else
+			outbuf[9] = 'T';
+	}
+
+	return outbuf;
+}
+
+/* read_symlink -- read a symbolic link into an allocated buffer.
+   This is based on xreadlink; the basic problem is that lstat cannot be relied
+   upon to return the proper size for a symbolic link.  This happens,
+   for example, on GNU/Linux in the /proc filesystem, where the symbolic link
+   sizes are often 0. */
+
+#ifndef SIZE_MAX
+# define SIZE_MAX ((size_t) -1)
+#endif
+#ifndef SSIZE_MAX
+# define SSIZE_MAX ((ssize_t) (SIZE_MAX / 2))
+#endif
+
+#define MAXSIZE (SIZE_MAX < SSIZE_MAX ? SIZE_MAX : SSIZE_MAX)
+
+static char *
+read_symlink(const char *fname, size_t bufsize, ssize_t *linksize)
+{
+	if (bufsize)
+		bufsize += 2;
+	else
+		bufsize = BUFSIZ * 2;
+
+	/* Make sure that bufsize >= 2 and within range */
+	if (bufsize > MAXSIZE || bufsize < 2)
+		bufsize = MAXSIZE;
+
+	while (1) {
+		char *buf;
+
+		emalloc(buf, char *, bufsize, "read_symlink");
+		if ((*linksize = readlink(fname, buf, bufsize)) < 0) {
+			/* On AIX 5L v5.3 and HP-UX 11i v2 04/09, readlink
+			   returns -1 with errno == ERANGE if the buffer is
+			   too small.  */
+			if (errno != ERANGE) {
+				free(buf);
+				return NULL;
+			}
+		}
+		/* N.B. This test is safe because bufsize must be >= 2 */
+		else if ((size_t)*linksize <= bufsize-2) {
+			buf[*linksize] = '\0';
+			return buf;
+		}
+		free(buf);
+		if (bufsize <= MAXSIZE/2)
+			bufsize *= 2;
+		else if (bufsize < MAXSIZE)
+			bufsize = MAXSIZE;
+		else
+			return NULL;
+	}
+	return NULL;
+}
+
+/* array_set --- set an array element */
+
+static void
+array_set(awk_array_t array, const char *sub, awk_value_t *value)
+{
+	awk_element_t element;
+
+	memset(& element, 0, sizeof(element));
+
+	emalloc(element.index.str, char *, strlen(sub), "array_set");
+
+	strcpy(element.index.str, sub);
+	element.index.len = strlen(sub);
+	element.value = *value;
+
+	set_array_element(ext_id, array, & element);
+}
+
+/* do_stat --- provide a stat() function for gawk */
+
+static int
+do_stat(int nargs)
+{
+	awk_value_t *file_param, *array_param;
+	char *name;
+	awk_array_t array;
+	struct stat sbuf;
+	int ret;
+	char *pmode;	/* printable mode */
+	char *type = "unknown";
+
+	if (do_lint && nargs != 2) {
+		lintwarn(ext_id, "stat: called with wrong number of arguments");
+		ret = 0;
+		goto out;
+	}
+
+	/* file is first arg, array to hold results is second */
+	file_param = get_curfunc_param(ext_id, 0, AWK_PARAM_STRING);
+	array_param = get_curfunc_param(ext_id, 0, AWK_PARAM_ARRAY);
+
+	if (file_param == NULL || array_param == NULL) {
+		warning(ext_id, "stat: bad paramaters");
+		ret = 0;
+		goto out;
+	}
+
+	name = file_param->str_value.str;
+	array = array_param->array_cookie;
+
+	/* empty out the array */
+	clear_array(ext_id, array);
+
+	/* lstat the file, if error, set ERRNO and return */
+	ret = lstat(name, & sbuf);
+	if (ret < 0) {
+		update_ERRNO_int(ext_id, errno);
+		ret = 0;
+		goto out;
+	}
+
+	/* fill in the array */
+	array_set(array, "name", make_string(ext_id, name, file_param->str_value.len));
+	array_set(array, "dev", make_number(ext_id, (double) sbuf.st_dev));
+	array_set(array, "ino", make_number(ext_id, (double) sbuf.st_ino));
+	array_set(array, "mode", make_number(ext_id, (double) sbuf.st_mode));
+	array_set(array, "nlink", make_number(ext_id, (double) sbuf.st_nlink));
+	array_set(array, "uid", make_number(ext_id, (double) sbuf.st_uid));
+	array_set(array, "gid", make_number(ext_id, (double) sbuf.st_gid));
+	array_set(array, "size", make_number(ext_id, (double) sbuf.st_size));
+	array_set(array, "blocks", make_number(ext_id, (double) sbuf.st_blocks));
+	array_set(array, "atime", make_number(ext_id, (double) sbuf.st_atime));
+	array_set(array, "mtime", make_number(ext_id, (double) sbuf.st_mtime));
+	array_set(array, "ctime", make_number(ext_id, (double) sbuf.st_ctime));
+
+	/* for block and character devices, add rdev, major and minor numbers */
+	if (S_ISBLK(sbuf.st_mode) || S_ISCHR(sbuf.st_mode)) {
+		array_set(array, "rdev", make_number(ext_id, (double) sbuf.st_rdev));
+		array_set(array, "major", make_number(ext_id, (double) major(sbuf.st_rdev)));
+		array_set(array, "minor", make_number(ext_id, (double) minor(sbuf.st_rdev)));
+	}
+
+#ifdef HAVE_ST_BLKSIZE
+	array_set(array, "blksize", make_number(ext_id, (double) sbuf.st_blksize));
+#endif /* HAVE_ST_BLKSIZE */
+
+	pmode = format_mode(sbuf.st_mode);
+	array_set(array, "pmode", make_string(ext_id, pmode, strlen(pmode)));
+
+	/* for symbolic links, add a linkval field */
+	if (S_ISLNK(sbuf.st_mode)) {
+		char *buf;
+		ssize_t linksize;
+
+		if ((buf = read_symlink(name, sbuf.st_size,
+					&linksize)) != NULL)
+			array_set(array, "linkval", make_string(ext_id, buf, linksize));
+		else
+			warning(ext_id, "unable to read symbolic link `%s'", name);
+	}
+
+	/* add a type field */
+	switch (sbuf.st_mode & S_IFMT) {
+#ifdef S_IFSOCK
+	case S_IFSOCK:
+		type = "socket";
+		break;
+#endif
+#ifdef S_IFLNK
+	case S_IFLNK:
+		type = "symlink";
+		break;
+#endif
+	case S_IFREG:
+		type = "file";
+		break;
+	case S_IFBLK:
+		type = "blockdev";
+		break;
+	case S_IFDIR:
+		type = "directory";
+		break;
+#ifdef S_IFDOOR
+	case S_IFDOOR:
+		type = "door";
+		break;
+#endif
+	case S_IFCHR:
+		type = "chardev";
+		break;
+#ifdef S_IFIFO
+	case S_IFIFO:
+		type = "fifo";
+		break;
+#endif
+	}
+
+	array_set(array, "type", make_string(ext_id, type, strlen(type)));
+
+	ret = 1;	/* success */
+
+out:
+	set_return_value(ext_id, make_number(ext_id, (double) ret));
+}
+
+
+/* dl_load --- load new builtins in this library */
+
+int dl_load(gawk_api_t *api_p, awk_ext_id_t id)
+{
+	static awk_ext_func_t func_table[] = {
+		{ "chdir", do_chdir, 1 },
+		{ "stat", do_stat, 2 },
+	};
+	size_t i, j;
+	int errors = 0;
+
+	api = api_p;
+	ext_id = id;
+
+	/* load functions */
+	for (i = 0, j = sizeof(func_table) / sizeof(func_table[0]); i < j; i++) {
+		if (! add_ext_func(ext_id, & func_table[i])) {
+			warning(ext_id, "filefuncs: could not add %s\n", func_table[i].name);
+			errors++;
+		}
+	}
+
+	return (errors == 0);
+}
diff --git a/gawkapi.c b/gawkapi.c
new file mode 100644
index 00000000..059d49e5
--- /dev/null
+++ b/gawkapi.c
@@ -0,0 +1,339 @@
+/*
+ * gawkapi.c -- Implement the functions defined for gawkapi.h
+ */
+
+/* 
+ * Copyright (C) 2012, the Free Software Foundation, Inc.
+ * 
+ * This file is part of GAWK, the GNU implementation of the
+ * AWK Programming Language.
+ * 
+ * GAWK is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * GAWK is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+ */
+
+#include "awk.h"
+
+/*
+ * Get the count'th paramater, zero-based.
+ * Returns NULL if count is out of range, or if actual paramater
+ * does not match what is specified in wanted.
+ */
+static awk_value_t *
+api_get_curfunc_param(awk_ext_id_t id, size_t count,
+			awk_param_type_t wanted)
+{
+	return NULL;	/* for now */
+}
+
+/* Set the return value. Gawk takes ownership of string memory */
+static void
+api_set_return_value(awk_ext_id_t id, const awk_value_t *retval)
+{
+}
+
+/* Functions to print messages */
+/* FIXME: Code duplicate from msg.c. Fix this. */
+static void
+api_fatal(awk_ext_id_t id, const char *format, ...)
+{
+	va_list args;
+	va_start(args, format);
+	err(_("fatal: "), format, args);
+	va_end(args);
+#ifdef GAWKDEBUG
+	abort();
+#endif
+	gawk_exit(EXIT_FATAL);
+}
+
+static void
+api_warning(awk_ext_id_t id, const char *format, ...)
+{
+	va_list args;
+	va_start(args, format);
+	err(_("warning: "), format, args);
+	va_end(args);
+}
+
+static void
+api_lintwarn(awk_ext_id_t id, const char *format, ...)
+{
+	va_list args;
+	va_start(args, format);
+	if (lintwarn == r_fatal) {
+		err(_("fatal: "), format, args);
+		va_end(args);
+#ifdef GAWKDEBUG
+		abort();
+#endif
+		gawk_exit(EXIT_FATAL);
+	} else {
+		err(_("warning: "), format, args);
+		va_end(args);
+	}
+}
+
+/* Register an open hook; for opening files read-only */
+static awk_bool_t
+api_register_open_hook(awk_ext_id_t id, void* (*open_func)(IOBUF *))
+{
+	return true;	/* for now */
+}
+
+/* Functions to update ERRNO */
+static void
+api_update_ERRNO_int(awk_ext_id_t id, int errno_val)
+{
+	update_ERRNO_int(errno_val);
+}
+
+static void
+api_update_ERRNO_string(awk_ext_id_t id, const char *string,
+		awk_bool_t translate)
+{
+	update_ERRNO_string(string, translate);
+}
+
+static void
+api_unset_ERRNO(awk_ext_id_t id)
+{
+	unset_ERRNO();
+}
+
+
+/* Add a function to the interpreter, returns true upon success */
+static awk_bool_t
+api_add_ext_func(awk_ext_id_t id, const awk_ext_func_t *func)
+{
+	return true;	/* for now */
+}
+
+/* Stuff for exit handler - do it as linked list */
+
+struct ext_exit_handler {
+	struct ext_exit_handler *next;
+	void (*funcp)(void *data, int exit_status);
+	void *arg0;
+};
+static struct ext_exit_handler *list_head = NULL;
+
+/* run_ext_exit_handlers --- run the extension exit handlers, LIFO order */
+
+void
+run_ext_exit_handlers(int exitval)
+{
+	struct ext_exit_handler *p, *next;
+
+	for (p = list_head; p != NULL; p = next) {
+		next = p->next;
+		p->funcp(p->arg0, exitval);
+		free(p);
+	}
+	list_head = NULL;
+}
+
+/* Add an exit call back, returns true upon success */
+static awk_bool_t
+api_awk_atexit(awk_ext_id_t id,
+		void (*funcp)(void *data, int exit_status),
+		void *arg0)
+{
+	struct ext_exit_handler *p;
+
+	/* allocate memory */
+	emalloc(p, struct ext_exit_handler *, sizeof(struct ext_exit_handler), "api_awk_atexit");
+
+	/* fill it in */
+	p->funcp = funcp;
+	p->arg0 = arg0;
+
+	/* add to linked list, LIFO order */
+	p->next = list_head;
+	list_head = p;
+	return true;	/* for now */
+}
+
+/*
+ * Symbol table access:
+ * 	- No access to special variables (NF, etc.)
+ * 	- One special exception: PROCINFO.
+ *	- Use sym_update() to change a value, including from UNDEFINED
+ *	  to scalar or array. 
+ */
+/*
+ * Lookup a variable, return its value. No messing with the value
+ * returned. Return value is NULL if the variable doesn't exist.
+ */
+static const awk_value_t *const
+api_sym_lookup(awk_ext_id_t id, const char *name)
+{
+	return NULL;	/* for now */
+}
+
+/*
+ * Update a value. Adds it to the symbol table if not there.
+ * Changing types is not allowed.
+ */
+static awk_bool_t
+api_sym_update(awk_ext_id_t id, const char *name, awk_value_t *value)
+{
+	return true;	/* for now */
+}
+
+/* Array management */
+/*
+ * Return the value of an element - read only!
+ * Use set_array_element to change it.
+ */
+static const awk_value_t *const
+api_get_array_element(awk_ext_id_t id,
+		awk_array_t a_cookie, const awk_value_t *const index)
+{
+	return NULL;	/* for now */
+}
+
+/*
+ * Change (or create) element in existing array with
+ * element->index and element->value.
+ */
+static awk_bool_t
+api_set_array_element(awk_ext_id_t id, awk_array_t a_cookie,
+				awk_element_t *element)
+{
+	return true;	/* for now */
+}
+
+/*
+ * Remove the element with the given index.
+ * Returns success if removed or if element did not exist.
+ */
+static awk_bool_t
+api_del_array_element(awk_ext_id_t id,
+		awk_array_t a_cookie, const awk_value_t* const index)
+{
+	return true;	/* for now */
+}
+
+/*
+ * Retrieve total number of elements in array.
+ * Returns false if some kind of error.
+ */
+static awk_bool_t
+api_get_element_count(awk_ext_id_t id,
+		awk_array_t a_cookie, size_t *count)
+{
+	return true;	/* for now */
+}
+
+/* Create a new array cookie to which elements may be added */
+static awk_array_t
+api_create_array(awk_ext_id_t id)
+{
+	return NULL;	/* for now */
+}
+
+/* Clear out an array */
+static awk_bool_t
+api_clear_array(awk_ext_id_t id, awk_array_t a_cookie)
+{
+	return true;	/* for now */
+}
+
+/* Flatten out an array so that it can be looped over easily. */
+static awk_bool_t
+api_flatten_array(awk_ext_id_t id,
+		awk_array_t a_cookie,
+		size_t *count,
+		awk_element_t **data)
+{
+	return true;	/* for now */
+}
+
+/*
+ * When done, release the memory, delete any marked elements
+ * Count must match what gawk thinks the size is.
+ */
+static awk_bool_t
+api_release_flattened_array(awk_ext_id_t id,
+		awk_array_t a_cookie,
+		size_t count,
+		awk_element_t *data)
+{
+	return true;	/* for now */
+}
+
+/* Constructor functions */
+static awk_value_t *
+api_make_string(awk_ext_id_t id,
+		const char *string, size_t length)
+{
+	static awk_value_t result;
+
+	result.val_type = AWK_STRING;
+	result.str_value.str = (char *) string;
+	result.str_value.len = length;
+
+	return & result;
+}
+
+static awk_value_t *
+api_make_number(awk_ext_id_t id, double num)
+{
+	static awk_value_t result;
+
+	result.val_type = AWK_NUMBER;
+	result.num_value = num;
+
+	return & result;
+}
+
+static gawk_api_t api_impl = {
+	GAWK_API_MAJOR_VERSION,	/* major and minor versions */
+	GAWK_API_MINOR_VERSION,
+	{ 0 },			/* do_flags */
+
+	api_get_curfunc_param,
+	api_set_return_value,
+
+	api_fatal,
+	api_warning,
+	api_lintwarn,
+
+	api_register_open_hook,
+
+	api_update_ERRNO_int,
+	api_update_ERRNO_string,
+	api_unset_ERRNO,
+
+	api_add_ext_func,
+
+	api_awk_atexit,
+
+	api_sym_lookup,
+	api_sym_update,
+
+	api_get_array_element,
+	api_set_array_element,
+	api_del_array_element,
+	api_get_element_count,
+	api_create_array,
+	api_clear_array,
+	api_flatten_array,
+	api_release_flattened_array,
+
+	/* Constructor functions */
+	api_make_string,
+	api_make_number,
+};
diff --git a/gawkapi.h b/gawkapi.h
index a09a10d7..2a14d334 100644
--- a/gawkapi.h
+++ b/gawkapi.h
@@ -31,11 +31,21 @@
 #ifndef _GAWK_API_H
 #define _GAWK_API_H
 
-/* Allow the use in C++ code.  */
+/*
+ * General introduction:
+ *
+ * This API purposely restricts itself to C90 features.
+ * In paticular, no bool, no // comments, no use of the
+ * restrict keyword, or anything else, in order to provide
+ * maximal portability.
+ */
+
+/* Allow use in C++ code.  */
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+/* struct used for reading records and managing buffers */
 typedef struct iobuf {
 	const char *name;       /* filename */
 	int fd;                 /* file descriptor */
@@ -58,7 +68,7 @@ typedef struct iobuf {
 
 	void *opaque;		/* private data for open hooks */
 	int (*get_record)(char **out, struct iobuf *, int *errcode);
-	void (*close_func)(struct iobuf *);		/* open and close hooks */
+	void (*close_func)(struct iobuf *);	/* open and close hooks */
 	
 	int errcode;
 
@@ -75,45 +85,92 @@ typedef struct iobuf {
 
 #define DO_FLAGS_SIZE	6
 
+/*
+ * This tag defines the type of a value.
+ * Values are associated with regular variables and with array elements.
+ * Since arrays can be multidimensional (as can regular variables)
+ * it's valid to have a "value" that is actually an array.
+ */
 typedef enum {
 	AWK_UNDEFINED,
 	AWK_NUMBER,
-	AWK_CONST_STRING,
 	AWK_STRING,
 	AWK_ARRAY
 } awk_valtype_t;
 
-typedef struct {
-	const char *const str;
-	const size_t len;
-} awk_const_string_t;
-
+/*
+ * A mutable string. Gawk owns the memory pointed to if it supplied
+ * the value. Otherwise, it takes ownership of the memory pointed to.
+ */
 typedef struct {
 	char *str;
 	size_t len;
 } awk_string_t;
 
+/* Arrays are represented as an opaque type */
+typedef void *awk_array_t;
+
+/*
+ * An awk value. The val_type tag indicates what
+ * is in the union.
+ */
 typedef struct {
 	awk_valtype_t	val_type;
 	union {
-		awk_const_string	cs;
-		awk_string		s;
-		double			d;
-		void*			a;
+		awk_string_t	s;
+		double		d;
+		awk_array_t	a;
 	} u;
-#define const_str_val	u.cs
-#define str_val		u.s
-#define num_val		u.d
+#define str_value	u.s
+#define num_value	u.d
 #define array_cookie	u.a
 } awk_value_t;
 
+/* possible kinds of function parameters */
+typedef enum {
+	AWK_PARAM_STRING,	/* expecting a scalar string */
+	AWK_PARAM_NUMBER,	/* expecting a scalar number */
+	AWK_PARAM_ARRAY,	/* expecting an array */
+} awk_param_type_t;
+
+/*
+ * A "flattened" array element. Gawk produces an array of these.
+ * ALL memory pointed to belongs to gawk. Individual elements may
+ * be marked for deletion. New elements must be added individually,
+ * one at a time, using the API for that purpose.
+ */
 
+typedef struct awk_element {
+	/* convenience linked list pointer, not used by gawk */
+	struct awk_element *next;
+	enum {
+		AWK_ELEMENT_DEFAULT = 0,	/* set by gawk */
+		AWK_ELEMENT_DELETE = 1		/* set by extension if
+						   should be deleted */
+	} flags;
+	awk_string_t	index;
+	awk_value_t	value;
+} awk_element_t;
+
+/*
+ * A record describing an extension function. Upon being
+ * loaded, the extension should pass in one of these for
+ * each C function.
+ */
 typedef struct {
 	const char *name;
-	size_t num_args;
-	void (*function)(int num_args);
+	int (*function)(int num_args_actual);
+	size_t num_args_expected;
 } awk_ext_func_t;
 
+typedef int awk_bool_t;	/* we don't use <stdbool.h> on purpose */
+
+typedef void *awk_ext_id_t;	/* opaque type for extension id */
+
+/*
+ * The API into gawk. Lots of functions here. We hope that they are
+ * logically organized.
+ */
 typedef struct gawk_api {
 	int major_version;
 	int minor_version;
@@ -127,46 +184,112 @@ typedef struct gawk_api {
 #define gawk_do_debug		4
 #define gawk_do_mpfr		5
 
-	/* get the number of arguments passed in function call */
-	/* FIXME: Needed? Won't we pass the count in the real call? */
-	size_t (*get_curfunc_arg_count)(void *ext_id);
-	awk_value_t *(*get_curfunc_param)(void *ext_id, size_t count);
+	/*
+	 * Get the count'th paramater, zero-based.
+	 * Returns NULL if count is out of range, or if actual paramater
+	 * does not match what is specified in wanted.
+	 */
+	awk_value_t *(*get_curfunc_param)(awk_ext_id_t id, size_t count,
+					awk_param_type_t wanted);
 
-	/* functions to print messages */
-	void (*fatal)(void *ext_id, const char *format, ...);
-	void (*warning)(void *ext_id, const char *format, ...);
-	void (*lintwarn)(void *ext_id, const char *format, ...);
+	/* Set the return value. Gawk takes ownership of string memory */
+	void (*set_return_value)(awk_ext_id_t id, const awk_value_t *retval);
 
-	/* register an open hook; for opening files read-only */
-	int (*register_open_hook)(void *ext_id,
-			void* (*open_func)(IOBUF *));
+	/* Functions to print messages */
+	void (*api_fatal)(awk_ext_id_t id, const char *format, ...);
+	void (*api_warning)(awk_ext_id_t id, const char *format, ...);
+	void (*api_lintwarn)(awk_ext_id_t id, const char *format, ...);
 
-	/* functions to update ERRNO */
-	void (*update_ERRNO_int)(void *ext_id, int);
-	void (*update_ERRNO_string)(void *ext_id, const char *string,
-			int translate);
-	void (*unset_ERRNO)(void *ext_id);
+	/* Register an open hook; for opening files read-only */
+	awk_bool_t (*register_open_hook)(awk_ext_id_t id, void* (*open_func)(IOBUF *));
 
-	/* check if a value received from gawk is the null string */
-	int (*is_null_string)(void *ext_id, void *value);
+	/* Functions to update ERRNO */
+	void (*update_ERRNO_int)(awk_ext_id_t id, int errno_val);
+	void (*update_ERRNO_string)(awk_ext_id_t id, const char *string,
+			awk_bool_t translate);
+	void (*unset_ERRNO)(awk_ext_id_t id);
 
-	/* add a function to the interpreter */
-	int *(add_ext_func)(void *ext_id, const awk_ext_func_t *func);
+	/* Add a function to the interpreter, returns true upon success */
+	awk_bool_t (*add_ext_func)(awk_ext_id_t id, const awk_ext_func_t *func);
 
-	/* add an exit call back */
-	void (*awk_atexit)(void *ext_id, void (*funcp)(void *data, int exit_status), void *arg0);
+	/* Add an exit call back, returns true upon success */
+	awk_bool_t (*awk_atexit)(awk_ext_id_t id,
+			void (*funcp)(void *data, int exit_status),
+			void *arg0);
+
+	/*
+	 * Symbol table access:
+	 * 	- No access to special variables (NF, etc.)
+	 * 	- One special exception: PROCINFO.
+	 *	- Use sym_update() to change a value, including from UNDEFINED
+	 *	  to scalar or array. 
+	 */
+	/*
+	 * Lookup a variable, return its value. No messing with the value
+	 * returned. Return value is NULL if the variable doesn't exist.
+	 */
+	const awk_value_t *const (*sym_lookup)(awk_ext_id_t id, const char *name);
 
-	/* Symbol table access */
-	awk_value_t *(*sym_lookup)(void *ext_id, const char *name);
-	int (*sym_update)(void *ext_id, const char *name, awk_value_t *value);
-	int (*sym_remove)(void *ext_id, const char *name);
+	/*
+	 * Update a value. Adds it to the symbol table if not there.
+	 * Changing types is not allowed.
+	 */
+	awk_bool_t (*sym_update)(awk_ext_id_t id, const char *name, awk_value_t *value);
 
 	/* Array management */
-	awk_value_t *(*get_array_element)(void *ext_id, void *a_cookie, const awk_value_t* const index);
-	awk_value_t *(*set_array_element)(void *ext_id, void *a_cookie,
-			const awk_value_t* const index, const awk_value_t* const value);
-	awk_value_t *(*del_array_element)(void *ext_id, void *a_cookie, const awk_value_t* const index);
-	size_t (*get_element_count)(void *ext_id, void *a_cookie);
+	/*
+	 * Return the value of an element - read only!
+	 * Use set_array_element to change it.
+	 */
+	const awk_value_t *const (*get_array_element)(awk_ext_id_t id,
+			awk_array_t a_cookie, const awk_value_t *const index);
+
+	/*
+	 * Change (or create) element in existing array with
+	 * element->index and element->value.
+	 */
+	awk_bool_t (*set_array_element)(awk_ext_id_t id, awk_array_t a_cookie,
+					awk_element_t *element);
+
+	/*
+	 * Remove the element with the given index.
+	 * Returns success if removed or if element did not exist.
+	 */
+	awk_bool_t (*del_array_element)(awk_ext_id_t id,
+			awk_array_t a_cookie, const awk_value_t* const index);
+
+	/*
+	 * Retrieve total number of elements in array.
+	 * Returns false if some kind of error.
+	 */
+	awk_bool_t (*get_element_count)(awk_ext_id_t id,
+			awk_array_t a_cookie, size_t *count);
+
+	/* Create a new array cookie to which elements may be added */
+	awk_array_t (*create_array)(awk_ext_id_t id);
+
+	/* Clear out an array */
+	awk_bool_t (*clear_array)(awk_ext_id_t id, awk_array_t a_cookie);
+
+	/* Flatten out an array so that it can be looped over easily. */
+	awk_bool_t (*flatten_array)(awk_ext_id_t id,
+			awk_array_t a_cookie,
+			size_t *count,
+			awk_element_t **data);
+
+	/*
+	 * When done, release the memory, delete any marked elements
+	 * Count must match what gawk thinks the size is.
+	 */
+	awk_bool_t (*release_flattened_array)(awk_ext_id_t id,
+			awk_array_t a_cookie,
+			size_t count,
+			awk_element_t *data);
+
+	/* Constructor functions */
+	awk_value_t *(*make_string)(awk_ext_id_t id,
+			const char *string, size_t length);
+	awk_value_t *(*make_number)(awk_ext_id_t id, double num);
 
 } gawk_api_t;
 
@@ -180,13 +303,14 @@ typedef struct gawk_api {
 #define do_profile	api->do_flags[gawk_do_profile]
 #define do_sandbox	api->do_flags[gawk_do_sandbox]
 #define do_debug	api->do_flags[gawk_do_debug]
+#define do_mpfr		api->do_flags[gawk_do_mpfr]
 
-#define get_curfunc_arg_count	api->get_curfunc_arg_count
 #define get_curfunc_param	api->get_curfunc_param
+#define set_return_value	api->set_return_value
 
-#define fatal		api->fatal
-#define warning		api->warning
-#define lintwarn	api->lintwarn
+#define fatal		api->api_fatal
+#define warning		api->api_warning
+#define lintwarn	api->api_lintwarn
 
 #define register_open_hook	api->register_open_hook
 
@@ -201,14 +325,37 @@ typedef struct gawk_api {
 
 #define sym_lookup	api->sym_lookup
 #define sym_update	api->sym_update
-#define sym_remove	api->sym_remove
 
 #define get_array_element	api->get_array_element
 #define set_array_element	api->set_array_element
 #define del_array_element	api->del_array_element
 #define get_element_count	api->get_element_count
+#define clear_array		api->clear_array
+#define create_array		api->create_array
+#define flatten_array		api->flatten_array
+#define release_flattened_array	api->release_flattened_array
+
+#define make_string	api->make_string
+#define make_number	api->make_number
+
+#define emalloc(pointer, type, size, message) \
+	do { \
+		if ((pointer = (type) malloc(size)) == 0) \
+			fatal(ext_id, "malloc of %d bytes failed\n", size); \
+	} while(0)
+
 #endif /* GAWK */
 
+/*
+ * Each extension must define a function with this prototype:
+ *
+ *	int dl_load(gawk_api_t *api_p, awk_ext_id_t id)
+ *
+ * For the macros to work, the function should save api_p in a
+ * global variable named 'api'. The return value should be zero
+ * on failure and non-zero on success.
+ */
+
 #ifdef __cplusplus
 }
 #endif	/* C++ */