Add fts() extension, support, doc, and test.

17 files changed, 728 insertions, 56 deletions

diff --git a/ChangeLog b/ChangeLog
index 1c686d0c..f0f5ff2b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -2,6 +2,10 @@
 
 	* configure.ac: Add -DNDEBUG to remove asserts if not developing.
 
+	* gawkapi.h: Document how to build up arrays.
+	* gawkapi.c (api_sym_update): For an array, pass the new cookie
+	back out to the extension.
+
 2012-08-01         Arnold D. Robbins     <arnold@skeeve.com>
 
 	* io.c (iop_finish): New function.
diff --git a/extension/ChangeLog b/extension/ChangeLog
index f5167c01..6b7c2bce 100644
--- a/extension/ChangeLog
+++ b/extension/ChangeLog
@@ -1,3 +1,13 @@
+2012-08-08         Arnold D. Robbins     <arnold@skeeve.com>
+
+	Add fts() to filefuncs.
+
+	* filefuncs.3am: Update doc.
+	* filefuncs.c: Lots of new code.
+	* configure.ac: Add checks for appropriate headers and functions.
+	* stack.h, stack.c: New files.
+	* Makefile.am: Update list of files.
+
 2012-08-03         Andrew J. Schorr     <aschorr@telemetry-investments.com>
 
 	* readdir.c (dir_get_record): Fix for systems where ino_t is
diff --git a/extension/Makefile.am b/extension/Makefile.am
index 2e6a523d..bf9a715f 100644
--- a/extension/Makefile.am
+++ b/extension/Makefile.am
@@ -46,7 +46,7 @@ MY_MODULE_FLAGS = -module -avoid-version -no-undefined
 # on Cygwin, gettext requires that we link with -lintl 
 MY_LIBS = $(LIBINTL)
 
-filefuncs_la_SOURCES  = filefuncs.c
+filefuncs_la_SOURCES  = filefuncs.c stack.h stack.c
 filefuncs_la_LDFLAGS  = $(MY_MODULE_FLAGS)
 filefuncs_la_LIBADD   = $(MY_LIBS)
 fnmatch_la_SOURCES    = fnmatch.c
diff --git a/extension/Makefile.in b/extension/Makefile.in
index e64fb88b..1679cabd 100644
--- a/extension/Makefile.in
+++ b/extension/Makefile.in
@@ -139,7 +139,7 @@ LTLIBRARIES = $(pkgextension_LTLIBRARIES)
 am__DEPENDENCIES_1 =
 am__DEPENDENCIES_2 = $(am__DEPENDENCIES_1)
 filefuncs_la_DEPENDENCIES = $(am__DEPENDENCIES_2)
-am_filefuncs_la_OBJECTS = filefuncs.lo
+am_filefuncs_la_OBJECTS = filefuncs.lo stack.lo
 filefuncs_la_OBJECTS = $(am_filefuncs_la_OBJECTS)
 filefuncs_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \
 	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
@@ -430,7 +430,7 @@ pkgextension_LTLIBRARIES = \
 MY_MODULE_FLAGS = -module -avoid-version -no-undefined
 # on Cygwin, gettext requires that we link with -lintl 
 MY_LIBS = $(LIBINTL)
-filefuncs_la_SOURCES = filefuncs.c
+filefuncs_la_SOURCES = filefuncs.c stack.h stack.c
 filefuncs_la_LDFLAGS = $(MY_MODULE_FLAGS)
 filefuncs_la_LIBADD = $(MY_LIBS)
 fnmatch_la_SOURCES = fnmatch.c
@@ -588,6 +588,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/readdir.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/readfile.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rwarray.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stack.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/testext.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/time.Plo@am__quote@
 
diff --git a/extension/configh.in b/extension/configh.in
index aae34842..73ce026c 100644
--- a/extension/configh.in
+++ b/extension/configh.in
@@ -23,6 +23,15 @@
 /* Define to 1 if you have the <fnmatch.h> header file. */
 #undef HAVE_FNMATCH_H
 
+/* Define to 1 if you have the <fts.h> header file. */
+#undef HAVE_FTS_H
+
+/* Define to 1 if you have the `fts_open' function. */
+#undef HAVE_FTS_OPEN
+
+/* Define to 1 if you have the `fts_read' function. */
+#undef HAVE_FTS_READ
+
 /* Define to 1 if you have the `GetSystemTimeAsFileTime' function. */
 #undef HAVE_GETSYSTEMTIMEASFILETIME
 
diff --git a/extension/configure b/extension/configure
index f7f6ab93..9335a25e 100755
--- a/extension/configure
+++ b/extension/configure
@@ -13438,7 +13438,7 @@ then
 	CFLAGS="$CFLAGS -Wall -Wextra"
 fi
 
-for ac_header in dirent.h fnmatch.h time.h sys/time.h sys/select.h
+for ac_header in dirent.h fnmatch.h fts.h time.h sys/time.h sys/select.h
 do :
   as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
 ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
@@ -13452,7 +13452,8 @@ fi
 done
 
 
-for ac_func in fdopendir fnmatch gettimeofday nanosleep select GetSystemTimeAsFileTime
+for ac_func in fdopendir fnmatch fts_open fts_read gettimeofday \
+		nanosleep select GetSystemTimeAsFileTime
 do :
   as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
 ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
diff --git a/extension/configure.ac b/extension/configure.ac
index 895572c6..33009611 100644
--- a/extension/configure.ac
+++ b/extension/configure.ac
@@ -48,9 +48,10 @@ then
 	CFLAGS="$CFLAGS -Wall -Wextra"
 fi
 
-AC_CHECK_HEADERS(dirent.h fnmatch.h time.h sys/time.h sys/select.h)
+AC_CHECK_HEADERS(dirent.h fnmatch.h fts.h time.h sys/time.h sys/select.h)
 
-AC_CHECK_FUNCS(fdopendir fnmatch gettimeofday nanosleep select GetSystemTimeAsFileTime)
+AC_CHECK_FUNCS(fdopendir fnmatch fts_open fts_read gettimeofday \
+		nanosleep select GetSystemTimeAsFileTime)
 
 dnl checks for compiler characteristics
 AC_C_INLINE
diff --git a/extension/filefuncs.3am b/extension/filefuncs.3am
index bdb8fe52..79694370 100644
--- a/extension/filefuncs.3am
+++ b/extension/filefuncs.3am
@@ -187,17 +187,22 @@ or
 .B FTS_PHYSICAL
 must be provided; otherwise
 .B fts()
-returns an error value.
+returns an error value and sets
+.BR ERRNO .
 .RS
 .TP
 .B FTS_LOGICAL
 Do a ``logical'' file traversal, where the information returned for
 a symbolic link refers to the linked-to file, and not to the
 symbolic link itself.
+This flag is mutually exclusive with
+.BR FTS_PHYSICAL .
 .TP
 .B FTS_PHYSICAL
 Do a ``physical'' file traversal, where the information returned for
 a symbolic link refers to the symbolic link itself.
+This flag is mutually exclusive with
+.BR FTS_LOGICAL .
 .TP
 .B FTS_NOCHDIR
 As a performance optimization, the
@@ -240,7 +245,7 @@ There are two cases.
 .RS
 .TP
 The path is a file.
-In this case, the array contains at two or three elements:
+In this case, the array contains two or three elements:
 .RS
 .TP
 \fB"path"\fP
diff --git a/extension/filefuncs.c b/extension/filefuncs.c
index e8c16e8f..e27e51bf 100644
--- a/extension/filefuncs.c
+++ b/extension/filefuncs.c
@@ -5,6 +5,7 @@
  * Arnold Robbins, update for 3.1, Mon Nov 23 12:53:39 EST 1998
  * Arnold Robbins and John Haque, update for 3.1.4, applied Mon Jun 14 13:55:30 IDT 2004
  * Arnold Robbins and Andrew Schorr, revised for new extension API, May 2012.
+ * Arnold Robbins, add fts(), August 2012
  */
 
 /*
@@ -46,9 +47,20 @@
 #define _(msgid)  gettext(msgid)
 #define N_(msgid) msgid
 
+#if defined(HAVE_FTS_H) && defined(HAVE_FTS_OPEN) && defined(HAVE_FTS_READ)
+#define HAVE_FTS_ROUTINES
+#endif
+
+
+#ifdef HAVE_FTS_ROUTINES
+#include <fts.h>
+#include "stack.h"
+#endif
+
 static const gawk_api_t *api;	/* for convenience macros to work */
 static awk_ext_id_t *ext_id;
-static awk_bool_t (*init_func)(void) = NULL;
+static awk_bool_t init_filefuncs(void);
+static awk_bool_t (*init_func)(void) = init_filefuncs;
 
 int plugin_is_GPL_compatible;
 
@@ -237,11 +249,10 @@ array_set_numeric(awk_array_t array, const char *sub, double num)
 /* fill_stat_array --- do the work to fill an array with stat info */
 
 static int
-fill_stat_array(const char *name, awk_array_t array)
+fill_stat_array(const char *name, awk_array_t array, struct stat *sbuf)
 {
 	char *pmode;	/* printable mode */
 	const char *type = "unknown";
-	struct stat sbuf;
 	awk_value_t tmp;
 	static struct ftype_map {
 		unsigned int mask;
@@ -264,52 +275,45 @@ fill_stat_array(const char *name, awk_array_t array)
 		{ S_IFDOOR, "door" },
 #endif /* S_IFDOOR */
 	};
-	int ret, j, k;
+	int j, k;
 
 	/* empty out the array */
 	clear_array(array);
 
-	/* lstat the file, if error, set ERRNO and return */
-	ret = lstat(name, & sbuf);
-	if (ret < 0) {
-		update_ERRNO_int(errno);
-		return -1;
-	}
-
 	/* fill in the array */
 	array_set(array, "name", make_const_string(name, strlen(name), & tmp));
-	array_set_numeric(array, "dev", sbuf.st_dev);
-	array_set_numeric(array, "ino", sbuf.st_ino);
-	array_set_numeric(array, "mode", sbuf.st_mode);
-	array_set_numeric(array, "nlink", sbuf.st_nlink);
-	array_set_numeric(array, "uid", sbuf.st_uid);
-	array_set_numeric(array, "gid", sbuf.st_gid);
-	array_set_numeric(array, "size", sbuf.st_size);
-	array_set_numeric(array, "blocks", sbuf.st_blocks);
-	array_set_numeric(array, "atime", sbuf.st_atime);
-	array_set_numeric(array, "mtime", sbuf.st_mtime);
-	array_set_numeric(array, "ctime", sbuf.st_ctime);
+	array_set_numeric(array, "dev", sbuf->st_dev);
+	array_set_numeric(array, "ino", sbuf->st_ino);
+	array_set_numeric(array, "mode", sbuf->st_mode);
+	array_set_numeric(array, "nlink", sbuf->st_nlink);
+	array_set_numeric(array, "uid", sbuf->st_uid);
+	array_set_numeric(array, "gid", sbuf->st_gid);
+	array_set_numeric(array, "size", sbuf->st_size);
+	array_set_numeric(array, "blocks", sbuf->st_blocks);
+	array_set_numeric(array, "atime", sbuf->st_atime);
+	array_set_numeric(array, "mtime", sbuf->st_mtime);
+	array_set_numeric(array, "ctime", sbuf->st_ctime);
 
 	/* for block and character devices, add rdev, major and minor numbers */
-	if (S_ISBLK(sbuf.st_mode) || S_ISCHR(sbuf.st_mode)) {
-		array_set_numeric(array, "rdev", sbuf.st_rdev);
-		array_set_numeric(array, "major", major(sbuf.st_rdev));
-		array_set_numeric(array, "minor", minor(sbuf.st_rdev));
+	if (S_ISBLK(sbuf->st_mode) || S_ISCHR(sbuf->st_mode)) {
+		array_set_numeric(array, "rdev", sbuf->st_rdev);
+		array_set_numeric(array, "major", major(sbuf->st_rdev));
+		array_set_numeric(array, "minor", minor(sbuf->st_rdev));
 	}
 
 #ifdef HAVE_ST_BLKSIZE
-	array_set_numeric(array, "blksize", sbuf.st_blksize);
+	array_set_numeric(array, "blksize", sbuf->st_blksize);
 #endif /* HAVE_ST_BLKSIZE */
 
-	pmode = format_mode(sbuf.st_mode);
+	pmode = format_mode(sbuf->st_mode);
 	array_set(array, "pmode", make_const_string(pmode, strlen(pmode), & tmp));
 
 	/* for symbolic links, add a linkval field */
-	if (S_ISLNK(sbuf.st_mode)) {
+	if (S_ISLNK(sbuf->st_mode)) {
 		char *buf;
 		ssize_t linksize;
 
-		if ((buf = read_symlink(name, sbuf.st_size,
+		if ((buf = read_symlink(name, sbuf->st_size,
 					& linksize)) != NULL)
 			array_set(array, "linkval", make_malloced_string(buf, linksize, & tmp));
 		else
@@ -319,7 +323,7 @@ fill_stat_array(const char *name, awk_array_t array)
 	/* add a type field */
 	type = "unknown";	/* shouldn't happen */
 	for (j = 0, k = sizeof(ftype_map)/sizeof(ftype_map[0]); j < k; j++) {
-		if ((sbuf.st_mode & S_IFMT) == ftype_map[j].mask) {
+		if ((sbuf->st_mode & S_IFMT) == ftype_map[j].mask) {
 			type = ftype_map[j].type;
 			break;
 		}
@@ -339,6 +343,7 @@ do_stat(int nargs, awk_value_t *result)
 	char *name;
 	awk_array_t array;
 	int ret;
+	struct stat sbuf;
 
 	assert(result != NULL);
 
@@ -357,14 +362,370 @@ do_stat(int nargs, awk_value_t *result)
 	name = file_param.str_value.str;
 	array = array_param.array_cookie;
 
-	ret = fill_stat_array(name, array);
+	/* lstat the file, if error, set ERRNO and return */
+	ret = lstat(name, & sbuf);
+	if (ret < 0) {
+		update_ERRNO_int(errno);
+		return make_number(ret, result);
+	}
+
+	ret = fill_stat_array(name, array, & sbuf);
 
 	return make_number(ret, result);
 }
 
+/* init_filefuncs --- initialization routine */
+
+static awk_bool_t
+init_filefuncs(void)
+{
+	int errors = 0;
+
+	/* at least right now, only FTS needs initializing */
+#ifdef HAVE_FTS_ROUTINES
+	int i;
+	awk_value_t value;
+
+	static struct flagtab {
+		const char *name;
+		int value;
+	} opentab[] = {
+#define ENTRY(x)	{ #x, x }
+		ENTRY(FTS_COMFOLLOW),
+		ENTRY(FTS_LOGICAL),
+		ENTRY(FTS_NOCHDIR),
+		ENTRY(FTS_PHYSICAL),
+		ENTRY(FTS_SEEDOT),
+		ENTRY(FTS_XDEV),
+		{ NULL, 0 }
+	};
+
+	for (i = 0; opentab[i].name != NULL; i++) {
+		(void) make_number(opentab[i].value, & value);
+		if (! sym_constant(opentab[i].name, & value)) {
+			warning(ext_id, "fts init: could not create constant %s",
+					opentab[i].name);
+			errors++;
+		}
+	}
+#endif
+	return errors == 0;
+}
+
+#ifdef HAVE_FTS_ROUTINES
+static int fts_errors = 0;
+
+/* fill_stat_element --- fill in stat element of array */
+
+static void
+fill_stat_element(awk_array_t element_array, const char *name, struct stat *sbuf)
+{
+	awk_value_t index, value;
+	awk_array_t stat_array;
+
+	stat_array = create_array();
+	if (stat_array == NULL) {
+		warning(ext_id, _("fill_stat_element: could not create array"));
+		fts_errors++;
+		return;
+	}
+	fill_stat_array(name, stat_array, sbuf);
+	(void) make_const_string("stat", 4, & index);
+	value.val_type = AWK_ARRAY;
+	value.array_cookie = stat_array;
+	if (! set_array_element(element_array, & index, & value)) {
+		warning(ext_id, _("fill_stat_element: could not set element"));
+		fts_errors++;
+	}
+}
+
+/* fill_path_element --- fill in path element of array */
+
+static void
+fill_path_element(awk_array_t element_array, const char *path)
+{
+	awk_value_t index, value;
+
+	(void) make_const_string("path", 4, & index);
+	(void) make_const_string(path, strlen(path), & value);
+	if (! set_array_element(element_array, & index, & value)) {
+		warning(ext_id, _("fill_path_element: could not set element"));
+		fts_errors++;
+	}
+}
+
+/* fill_error_element --- fill in error element of array */
+
+static void
+fill_error_element(awk_array_t element_array, const int errcode)
+{
+	awk_value_t index, value;
+	const char *err = strerror(errcode);
+
+	(void) make_const_string("error", 5, & index);
+	(void) make_const_string(err, strlen(err), & value);
+	if (! set_array_element(element_array, & index, & value)) {
+		warning(ext_id, _("fill_error_element: could not set element"));
+		fts_errors++;
+	}
+}
+
+/* fill_default_elements --- fill in stat and path elements */
+
+static void
+fill_default_elements(awk_array_t element_array, const FTSENT *const fentry, int bad_ret)
+{
+	/* full path */
+	fill_path_element(element_array, fentry->fts_path);
+
+	/* stat info */
+	if (! bad_ret) {
+		fill_stat_element(element_array,
+				fentry->fts_name,
+				fentry->fts_statp);
+	}
+
+	/* error info */
+	if (bad_ret || fentry->fts_errno != 0) {
+		fill_error_element(element_array, fentry->fts_errno);
+	}
+}
+
+/* process --- process the heirarchy */
+
+static void
+process(FTS *heirarchy, awk_array_t destarray, int seedot)
+{
+	FTSENT *fentry;
+	awk_value_t index, value;
+	awk_array_t element_array, newdir_array, dot_array;
+	int bad_ret = 0;
+
+	/* path is full path,  pathlen is length thereof */
+	/* name is name in directory, namelen is length thereof */
+	while ((fentry = fts_read(heirarchy)) != NULL) {
+		bad_ret = 0;
+
+		switch (fentry->fts_info) {
+		case FTS_D:
+			/* directory */
+			/* create array to hold entries */
+			newdir_array = create_array();
+			if (newdir_array == NULL) {
+				warning(ext_id, _("fts-process: could not create array"));
+				fts_errors++;
+				break;
+			}
+
+			/* store new directory in its parent directory */
+			(void) make_const_string(fentry->fts_name, fentry->fts_namelen, & index);
+			value.val_type = AWK_ARRAY;
+			value.array_cookie = newdir_array;
+			if (! set_array_element(destarray, & index, & value)) {
+				warning(ext_id, _("fts-process: could not set element"));
+				fts_errors++;
+				break;
+			}
+			newdir_array = value.array_cookie;
+
+			/* push current directory */
+			stack_push(destarray);
+
+			/* new directory becomes current */
+			destarray = newdir_array;
+			break;
+
+		case FTS_DNR:
+		case FTS_DC:
+		case FTS_ERR:
+		case FTS_NS:
+			/* error */
+			bad_ret = 1;
+			/* fall through */
+
+		case FTS_NSOK:
+		case FTS_SL:
+		case FTS_SLNONE:
+		case FTS_F:
+		case FTS_DOT:
+			/* if see dot, skip "." */
+			if (seedot && strcmp(fentry->fts_name, ".") == 0)
+				break;
+
+			/*
+			 * File case.
+			 * destarray is the directory we're reading.
+			 * step 1: create new empty array
+			 */
+			element_array = create_array();
+			if (element_array == NULL) {
+				warning(ext_id, _("fts-process: could not create array"));
+				fts_errors++;
+				break;
+			}
+
+			/* step 2: add element array to parent array */
+			(void) make_const_string(fentry->fts_name, fentry->fts_namelen, & index);
+			value.val_type = AWK_ARRAY;
+			value.array_cookie = element_array;
+			if (! set_array_element(destarray, & index, & value)) {
+				warning(ext_id, _("fts-process: could not set element"));
+				fts_errors++;
+				break;
+			}
+
+			/* step 3: fill in path, stat, error elements */
+			fill_default_elements(element_array, fentry, bad_ret);
+			break;
+
+		case FTS_DP:
+			/* create "." subarray */
+			dot_array = create_array();
+
+			/* add it to parent */
+			(void) make_const_string(".", 1, & index);
+			value.val_type = AWK_ARRAY;
+			value.array_cookie = dot_array;
+			if (! set_array_element(destarray, & index, & value)) {
+				warning(ext_id, _("fts-process: could not set element"));
+				fts_errors++;
+				break;
+			}
+
+			/* fill it in with path, stat, error elements */
+			fill_default_elements(dot_array, fentry, bad_ret);
+
+			/* now pop the parent directory off the stack */
+			if (! stack_empty()) {
+				/* pop stack */
+				destarray = stack_pop();
+			}
+
+			break;
+
+		case FTS_DEFAULT:
+			/* nothing to do */
+			break;
+		}
+	}
+}
+#endif
+
+/*  do_fts --- walk a heirarchy and fill in an array */
+
+/*
+ * Usage from awk:
+ *	flags = or(FTS_PHYSICAL, ...)
+ *	result = fts(pathlist, flags, filedata)
+ */
+
+static awk_value_t *
+do_fts(int nargs, awk_value_t *result)
+{
+#ifdef HAVE_FTS_ROUTINES
+	awk_value_t pathlist, flagval, dest;
+	awk_flat_array_t *path_array = NULL;
+	char **pathvector = NULL;
+	FTS *heirarchy;
+	int flags;
+	size_t i, count;
+	int ret = -1;
+	static const int mask = (
+		  FTS_COMFOLLOW | FTS_LOGICAL | FTS_NOCHDIR | FTS_PHYSICAL
+		| FTS_SEEDOT | FTS_XDEV);
+
+	assert(result != NULL);
+	fts_errors = 0;		/* ensure a fresh start */
+
+	if (do_lint && nargs != 3)
+		lintwarn(ext_id, _("fts: called with incorrect number of arguments, expecting 3"));
+
+	if (! get_argument(0, AWK_ARRAY, & pathlist)) {
+		warning(ext_id, _("fts: bad first parameter"));
+		update_ERRNO_int(EINVAL);
+		goto out;
+	}
+
+	if (! get_argument(1, AWK_NUMBER, & flagval)) {
+		warning(ext_id, _("fts: bad second parameter"));
+		update_ERRNO_int(EINVAL);
+		goto out;
+	}
+
+	if (! get_argument(2, AWK_ARRAY, & dest)) {
+		warning(ext_id, _("fts: bad third parameter"));
+		update_ERRNO_int(EINVAL);
+		goto out;
+	}
+
+	/* flatten pathlist */
+	if (! flatten_array(pathlist.array_cookie, & path_array)) {
+		warning(ext_id, _("fts: could not flatten array\n"));
+		goto out;
+	}
+
+	/* check the flags first, before the array flattening */
+
+	/* get flags */
+	flags = flagval.num_value;
+
+	/* enforce physical or logical but not both, and not no_stat */
+	if ((flags & (FTS_PHYSICAL|FTS_LOGICAL)) == 0
+	    || (flags & (FTS_PHYSICAL|FTS_LOGICAL)) == (FTS_PHYSICAL|FTS_LOGICAL)) {
+		update_ERRNO_int(EINVAL);
+		goto out;
+	}
+	if ((flags & FTS_NOSTAT) != 0) {
+		flags &= ~FTS_NOSTAT;
+		if (do_lint)
+			lintwarn(ext_id, _("fts: ignoring sneaky FTS_NOSTAT flag. nyah, nyah, nyah."));
+	}
+	flags &= mask;	/* turn off anything else */
+
+	/* make pathvector */
+	count = path_array->count + 1;
+	emalloc(pathvector, char **, count * sizeof(char *), "do_fts");
+	memset(pathvector, 0, count * sizeof(char *));
+
+	/* fill it in */
+	count--;	/* ignore final NULL at end of vector */
+	for (i = 0; i < count; i++)
+		pathvector[i] = path_array->elements[i].value.str_value.str;
+
+
+	/* clear dest array */
+	if (! clear_array(dest.array_cookie)) {
+		warning(ext_id, _("fts: clear_array failed\n"));
+		goto out;
+	}
+
+	/* let's do it! */
+	if ((heirarchy = fts_open(pathvector, flags, NULL)) != NULL) {
+		process(heirarchy, dest.array_cookie, (flags & FTS_SEEDOT) != 0);
+		fts_close(heirarchy);
+
+		if (fts_errors == 0)
+			ret = 0;
+	} else
+		update_ERRNO_int(errno);
+
+out:
+	if (pathvector != NULL)
+		free(pathvector);
+	if (path_array != NULL)
+		(void) release_flattened_array(pathlist.array_cookie, path_array);
+
+	return make_number(ret, result);
+#else
+	update_ERRNO_int(EINVAL);
+	return make_number(-1, result);
+#endif
+}
+
 static awk_ext_func_t func_table[] = {
-	{ "chdir", do_chdir, 1 },
-	{ "stat", do_stat, 2 },
+	{ "chdir",	do_chdir, 1 },
+	{ "stat",	do_stat, 2 },
+	{ "fts",	do_fts, 3 },
 };
 
 
diff --git a/extension/stack.c b/extension/stack.c
new file mode 100644
index 00000000..ec994c61
--- /dev/null
+++ b/extension/stack.c
@@ -0,0 +1,90 @@
+/*
+ * stack.c -- Implementation for stack functions for use by extensions.
+ */
+
+/* 
+ * Copyright (C) 2012 the Free Software Foundation, Inc.
+ * 
+ * This file is part of GAWK, the GNU implementation of the
+ * AWK Programming Language.
+ * 
+ * GAWK is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * GAWK is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+ */
+
+#include <stdlib.h>
+
+#include "stack.h"
+
+#define INITIAL_STACK	20
+
+static size_t size;
+static void **stack;
+static int index = -1;
+
+/* stack_empty --- return true if stack is empty */
+
+int
+stack_empty()
+{
+	return index < 0;
+}
+
+/* stack_top --- return top object on the stack */
+
+void *
+stack_top()
+{
+	if (stack_empty() || stack == NULL)
+		return NULL;
+
+	return stack[index];
+}
+
+/* stack_pop --- pop top object and return it */
+
+void *
+stack_pop()
+{
+	if (stack_empty() || stack == NULL)
+		return NULL;
+
+	return stack[index--];
+}
+
+/* stack_push --- push an object onto the stack */
+
+int stack_push(void *object)
+{
+	void **new_stack;
+	size_t new_size = 2 * size;
+
+	if (stack == NULL) {
+		stack = (void **) malloc(INITIAL_STACK * sizeof(void *));
+		if (stack == NULL)
+			return 0;
+		size = INITIAL_STACK;
+	} else if (index + 1 >= size) {
+		if (new_size < size)
+			return 0;
+		new_stack = realloc(stack, new_size * sizeof(void *));
+		if (new_stack == NULL)
+			return 0;
+		size = new_size;
+		stack = new_stack;
+	}
+
+	stack[++index] = object;
+	return 1;
+}
diff --git a/extension/stack.h b/extension/stack.h
new file mode 100644
index 00000000..8fc06e78
--- /dev/null
+++ b/extension/stack.h
@@ -0,0 +1,31 @@
+/*
+ * stack.h -- Definitions for stack functions for use by extensions.
+ */
+
+/* 
+ * Copyright (C) 2012 the Free Software Foundation, Inc.
+ * 
+ * This file is part of GAWK, the GNU implementation of the
+ * AWK Programming Language.
+ * 
+ * GAWK is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * GAWK is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+ */
+
+extern int stack_empty();	/* return true if stack is empty */
+extern void *stack_top();	/* return top object on the stack */
+extern void *stack_pop();	/* pop top object and return it */
+extern int stack_push(void *);	/* push an object onto the stack,
+				 * return 0 if failed, 1 if success 
+				 */
diff --git a/gawkapi.c b/gawkapi.c
index 721d69dc..c34b52bb 100644
--- a/gawkapi.c
+++ b/gawkapi.c
@@ -517,6 +517,7 @@ sym_update_real(awk_ext_id_t id,
 			array_node->vname = node->vname;
 			*node = *array_node;
 			freenode(array_node);
+			value->array_cookie = node;	/* pass new cookie back to extension */
 		} else {
 			/* regular variable */
 			node = install_symbol(estrdup((char *) name, strlen(name)),
diff --git a/gawkapi.h b/gawkapi.h
index 7e20d3d1..f345d07a 100644
--- a/gawkapi.h
+++ b/gawkapi.h
@@ -51,17 +51,37 @@
  * and reuse it directly, even for something that is conceptually pass
  * by value.
  *
- * 2. The correct way to create new arrays is to work "bottom up".
+ * 2. Due to gawk internals, after using sym_update() to install an array
+ * into gawk, you have to retrieve the array cookie from the value
+ * passed in to sym_update().  Like so:
  *
  *	new_array = create_array();
- *	// fill in new array with lots of subscripts and values
  *	val.val_type = AWK_ARRAY;
  *	val.array_cookie = new_array;
- *	sym_update("array", &val);	// install array in the symbol table
+ *	sym_update("array", & val);	// install array in the symbol table
+ *
+ *	new_array = val.array_cookie;	// MUST DO THIS
+ *
+ *	// fill in new array with lots of subscripts and values
+ *
+ * Similarly, if installing a new array as a subarray of an existing
+ * array, you must add the new array to its parent before adding any
+ * elements to it.
+ *
+ * You must also retrieve the value of the array_cookie after the call
+ * to set_element().
+ *
+ * Thus, the correct way to build an array is to work "top down".
+ * Create the array, and immediately install it in gawk's symbol table
+ * using sym_update(), or install it as an element in a previously
+ * existing array using set_element().
  *
- * After doing so, do NOT make further use of the new_array variable;
- * instead use sym_lookup to get the array_cookie if you need to do further
- * manipulation of the array.
+ * The new array must ultimately be rooted in a global symbol. This is
+ * necessary before installing any subarrays in it, due to gawk's
+ * internal implementation.  Strictly speaking, this is required only
+ * for arrays that will have subarrays as elements; however it is
+ * a good idea to always do this.  This restriction may be relaxed
+ * in a subsequent revision of the API.
  */
 
 /* Allow use in C++ code.  */
@@ -249,15 +269,15 @@ typedef void *awk_ext_id_t;	/* opaque type for extension id */
  * logically organized.
  */
 typedef struct gawk_api {
-	int major_version;
-	int minor_version;
+	awk_const int major_version;
+	awk_const int minor_version;
 
 	/*
 	 * These can change on the fly as things happen within gawk.
 	 * Currently only do_lint is prone to change, but we reserve
 	 * the right to allow the others also.
 	 */
-	int do_flags[DO_FLAGS_SIZE];
+	awk_const int do_flags[DO_FLAGS_SIZE];
 /* Use these as indices into do_flags[] array to check the values */
 #define gawk_do_lint		0
 #define gawk_do_traditional	1
diff --git a/test/ChangeLog b/test/ChangeLog
index c012513a..2b1ac648 100644
--- a/test/ChangeLog
+++ b/test/ChangeLog
@@ -1,3 +1,8 @@
+2012-08-08         Arnold D. Robbins     <arnold@skeeve.com>
+
+	* Makefile.am (fts): New test.
+	* fts.awk: New file.
+
 2012-07-30         Arnold D. Robbins     <arnold@skeeve.com>
 
 	* Makefile.am (assignconst): Use AWKPATH to get results that will
diff --git a/test/Makefile.am b/test/Makefile.am
index 270cc2fc..3a8e48ca 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -275,6 +275,7 @@ EXTRA_DIST = \
 	fstabplus.awk \
 	fstabplus.in \
 	fstabplus.ok \
+	fts.awk \
 	funlen.awk \
 	funlen.in \
 	funlen.ok \
@@ -895,7 +896,7 @@ LOCALE_CHARSET_TESTS = \
 	mbprintf1 mbprintf2 mbprintf3 rebt8b2 rtlenmb sort1 sprintfc
 
 SHLIB_TESTS = \
-	assignconst fnmatch filefuncs fork fork2 ordchr ordchr2 \
+	assignconst fnmatch filefuncs fork fork2 fts ordchr ordchr2 \
 	readdir readfile rwarray testext time
 
 # List of the tests which should be run with --lint option:
@@ -1612,6 +1613,11 @@ readdir:
 	@$(AWK) -f $(srcdir)/readdir.awk . > $@.out2
 	@-$(CMP) $@.out1 $@.out2 && rm -f $@.out[12]
 
+fts:
+	@echo $@
+	@$(AWK) -f $(srcdir)/fts.awk
+	@-$(CMP) $@.out1 $@.out2 && rm -f $@.out[12]
+
 # Targets generated for other tests:
 include Maketests
 
@@ -1620,7 +1626,7 @@ $(srcdir)/Maketests: $(srcdir)/Makefile.am $(srcdir)/Gentests
 	$(AWK) -f $(srcdir)/Gentests "$(srcdir)/Makefile.am" $$files > $(srcdir)/Maketests
 
 clean:
-	rm -fr _* core core.* fmtspcl.ok junk out1 out2 out3 strftime.ok test1 test2 seq *~ readfile.ok fork.tmp.* testext.awk
+	rm -fr _* core core.* fmtspcl.ok junk out1 out2 out3 strftime.ok test1 test2 seq *~ readfile.ok fork.tmp.* testext.awk fts.out1 fts.out2
 
 # An attempt to print something that can be grepped for in build logs
 pass-fail:
diff --git a/test/Makefile.in b/test/Makefile.in
index 4b6d5439..0ba7c5c5 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -488,6 +488,7 @@ EXTRA_DIST = \
 	fstabplus.awk \
 	fstabplus.in \
 	fstabplus.ok \
+	fts.awk \
 	funlen.awk \
 	funlen.in \
 	funlen.ok \
@@ -1104,7 +1105,7 @@ LOCALE_CHARSET_TESTS = \
 	mbprintf1 mbprintf2 mbprintf3 rebt8b2 rtlenmb sort1 sprintfc
 
 SHLIB_TESTS = \
-	assignconst fnmatch filefuncs fork fork2 ordchr ordchr2 \
+	assignconst fnmatch filefuncs fork fork2 fts ordchr ordchr2 \
 	readdir readfile rwarray testext time
 
 
@@ -1994,6 +1995,11 @@ readdir:
 	@ls -fli | sed 1d | $(AWK) -f $(srcdir)/readdir0.awk > $@.out1
 	@$(AWK) -f $(srcdir)/readdir.awk . > $@.out2
 	@-$(CMP) $@.out1 $@.out2 && rm -f $@.out[12]
+
+fts:
+	@echo $@
+	@$(AWK) -f $(srcdir)/fts.awk
+	@-$(CMP) $@.out1 $@.out2 && rm -f $@.out[12]
 Gt-dummy:
 # file Maketests, generated from Makefile.am by the Gentests program
 addcomma:
@@ -3239,7 +3245,7 @@ $(srcdir)/Maketests: $(srcdir)/Makefile.am $(srcdir)/Gentests
 	$(AWK) -f $(srcdir)/Gentests "$(srcdir)/Makefile.am" $$files > $(srcdir)/Maketests
 
 clean:
-	rm -fr _* core core.* fmtspcl.ok junk out1 out2 out3 strftime.ok test1 test2 seq *~ readfile.ok fork.tmp.* testext.awk
+	rm -fr _* core core.* fmtspcl.ok junk out1 out2 out3 strftime.ok test1 test2 seq *~ readfile.ok fork.tmp.* testext.awk fts.out1 fts.out2
 
 # An attempt to print something that can be grepped for in build logs
 pass-fail:
diff --git a/test/fts.awk b/test/fts.awk
new file mode 100644
index 00000000..b90adfb5
--- /dev/null
+++ b/test/fts.awk
@@ -0,0 +1,121 @@
+@load "filefuncs"
+
+BEGIN {
+	Level = 0
+
+	system("rm -fr d1 d2")
+	system("mkdir d1 d2 ; touch d1/f1 d1/f2 d2/f1 d2/f2")
+	pathlist[1] = "d1"
+	pathlist[2] = "d2"
+	flags = FTS_PHYSICAL
+	fts(pathlist, flags, data)
+
+	output = "fts.out1"
+	traverse(data)
+	close(output)
+
+	ftswalk(pathlist, data2)
+	output = "fts.out2"
+	traverse(data2)
+	close(output)
+
+	system("rm -fr d1 d2")
+}
+
+function indent(        i)
+{
+	for (i = 1; i <= Level; i++)
+		printf("\t") > output
+}
+
+function sort_traverse(data,	sorted, i)
+{
+	asorti(data, sorted)
+	for (i = 1; i in sorted; i++) {
+		indent()
+		printf("%s --> %s\n", sorted[i], data[sorted[i]]) > output
+	}
+}
+
+function traverse(data,         i)
+{
+	for (i in data) {
+		if (isarray(data[i])) {
+			indent()
+			printf("%s:\n", i) > output
+
+			Level++
+			if (("mtime" in data[i]) && ! isarray(data[i][mtime])) {
+				sort_traverse(data[i])
+			} else {
+				traverse(data[i])
+			}
+			Level--
+		} else {
+			indent()
+			printf("%s --> %s\n", i, data[i]) > output
+		}
+	}
+}
+
+
+function ftswalk(pathlist, data,	i, toppath)
+{
+	delete data
+	for (i = 1; i in pathlist; i++) {
+		toppath = pathlist[i]
+		data[toppath]["junk"]++	# create array
+		delete data[toppath]["junk"]
+		process(pathlist[i], data)
+	}
+}
+
+# enter process with pathname, array for that path already created but
+# empty
+
+function process(pathname, data_array,
+		stat_data, i, direntry, command, shortname)	# locals
+{
+	stat(pathname, stat_data)
+	if (stat_data["type"] == "file") {
+		shortname = strrstr(pathname, "/")
+		data_array["path"] = pathname
+		for (i in stat_data) {
+			if (i == "name")
+				data_array["stat"][i] = shortname
+			else
+				data_array["stat"][i] = stat_data[i]
+		}
+
+		return
+	}
+
+	# stuff for a directory
+
+	data_array[pathname]["."]["path"] = pathname
+	for (i in stat_data)
+		data_array[pathname]["."]["stat"][i] = stat_data[i]
+
+	command = ("ls -f " pathname)
+	while ((command | getline direntry) > 0) {
+		if (direntry == "." || direntry == "..")
+			continue
+		data_array[pathname][direntry]["junk"]++
+		delete data_array[pathname][direntry]["junk"]
+		process(pathname "/" direntry,
+			data_array[pathname][direntry])
+	}
+	close(command)
+}
+
+function strrstr(string, delim,		ind)
+{
+	if ((ind = index(string, delim)) == 0)
+		return string
+
+	do {
+		string = substr(string, ind + 1)
+	} while ((ind = index(string, delim)) > 0)
+
+	return string
+}