Enhance get_file API to return info about input and output and to enable extensions to create already-opened files or sockets.

12 files changed, 201 insertions, 39 deletions

diff --git a/ChangeLog b/ChangeLog
index f37300d0..86477a6b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,26 @@
+2014-11-06         Andrew J. Schorr     <aschorr@telemetry-investments.com>
+
+	* awk.h (redirect_string): First argument should be const.  Add a new
+	extfd argument to enable extensions to create files with pre-opened
+	file descriptors.
+	(after_beginfile): Declare function used in both eval.c and gawkapi.c.
+	* eval.c (after_beginfile): Remove extern declaration now in awk.h.
+	* gawkapi.c (api_get_file): Implement API changes to return
+	awk_input_buf_t and/or awk_output_buf_t info, as well as accept an
+	fd for inserting an opened file into the table.
+	* gawkapi.h (gawk_api): Modify the api_get_file declaration to
+	return awk_bool_t and add 3 new arguments -- a file descriptor
+	for inserting an already opened file, and awk_input_buf_t and
+	awk_output_buf_t to return info about both input and output.
+	(get_file): Add new arguments to the macro.
+	* io.c (redirect_string): First arg should be const, and add a new
+	extfd arg so extensions can pass in a file that has already been
+	opened by the extension.  Use the passed-in fd when appropriate,
+	and pass it into two_way_open.
+	(redirect): Pass new fd -1 arg to redirect_string.
+	(two_way_open): Accept new extension fd parameter and open it
+	as a socket.
+
 2014-11-05         Andrew J. Schorr     <aschorr@telemetry-investments.com>
 
 	* io.c (retryable): New function to indicate whether I/O can be
diff --git a/awk.h b/awk.h
index 1f4364ad..8ec6bccb 100644
--- a/awk.h
+++ b/awk.h
@@ -1524,7 +1524,7 @@ extern void set_FNR(void);
 extern void set_NR(void);
 
 extern struct redirect *redirect(NODE *redir_exp, int redirtype, int *errflg);
-extern struct redirect *redirect_string(char *redir_exp_str, size_t redir_exp_len, int not_string_flag, int redirtype, int *errflg);
+extern struct redirect *redirect_string(const char *redir_exp_str, size_t redir_exp_len, int not_string_flag, int redirtype, int *errflg, int extfd);
 extern NODE *do_close(int nargs);
 extern int flush_io(void);
 extern int close_io(bool *stdio_problem);
@@ -1543,6 +1543,7 @@ extern int is_off_limits_var(const char *var);
 extern char *estrdup(const char *str, size_t len);
 extern void update_global_values();
 extern long getenv_long(const char *name);
+extern void after_beginfile(IOBUF **curfile);
 
 /* mpfr.c */
 extern void set_PREC(void);
diff --git a/eval.c b/eval.c
index e7a346d3..2c8d3064 100644
--- a/eval.c
+++ b/eval.c
@@ -25,7 +25,6 @@
 
 #include "awk.h"
 
-extern void after_beginfile(IOBUF **curfile);
 extern double pow(double x, double y);
 extern double modf(double x, double *yp);
 extern double fmod(double x, double y);
diff --git a/extension/ChangeLog b/extension/ChangeLog
index 5d278f6c..b8e68674 100644
--- a/extension/ChangeLog
+++ b/extension/ChangeLog
@@ -1,3 +1,15 @@
+2014-11-06         Andrew J. Schorr     <aschorr@telemetry-investments.com>
+
+	* errno.c (do_errno2name, do_name2errno): Remove unused variable 'str'.
+	* select.c (do_signal): Remove unused variable 'override'.
+	(grabfd): New helper function to map a gawk file to the appropriate
+	fd for use in the arguments to selectd.
+	(do_select): get_file has 3 new arguments and returns info about both
+	the input and output buf.
+	(do_set_non_blocking): Support changes to get_file API.
+	* testext.c (test_get_file): New test function to check that extension
+	file creation via the get_file API is working.
+
 2014-11-05         Andrew J. Schorr     <aschorr@telemetry-investments.com>
 
 	* select.c (set_retry): New function to set PROCINFO[<name>, "RETRY"].
diff --git a/extension/errno.c b/extension/errno.c
index 2eafa437..5dc15d79 100644
--- a/extension/errno.c
+++ b/extension/errno.c
@@ -86,7 +86,6 @@ static awk_value_t *
 do_errno2name(int nargs, awk_value_t *result)
 {
 	awk_value_t errnum;
-	const char *str;
 
 	if (do_lint && nargs > 1)
 		lintwarn(ext_id, _("errno2name: called with too many arguments"));
@@ -112,7 +111,6 @@ static awk_value_t *
 do_name2errno(int nargs, awk_value_t *result)
 {
 	awk_value_t err;
-	const char *str;
 
 	if (do_lint && nargs > 1)
 		lintwarn(ext_id, _("name2errno: called with too many arguments"));
diff --git a/extension/select.c b/extension/select.c
index 1752ee34..597b3a6b 100644
--- a/extension/select.c
+++ b/extension/select.c
@@ -223,7 +223,6 @@ do_signal(int nargs, awk_value_t *result)
 	
 #ifdef HAVE_SIGACTION
 	{
-		awk_value_t override;
 		struct sigaction sa, prev;
 		sa.sa_handler = func;
 		sigfillset(& sa.sa_mask);  /* block all signals in handler */
@@ -307,6 +306,27 @@ do_kill(int nargs, awk_value_t *result)
 #endif
 }
 
+static int
+grabfd(int i, const awk_input_buf_t *ibuf, const awk_output_buf_t *obuf, const char *fnm, const char *ftp)
+{
+	switch (i) {
+	case 0:	/* read */
+		return ibuf ? ibuf->fd : -1;
+	case 1:	/* write */
+		return obuf ? fileno(obuf->fp) : -1;
+	case 2: /* except */
+		if (ibuf) {
+			if (obuf && ibuf->fd != fileno(obuf->fp))
+				warning(ext_id, _("select: `%s', `%s' in `except' array has clashing fds, using input %d, not output %d"), fnm, ftp, ibuf->fd, fileno(obuf->fp));
+			return ibuf->fd;
+		}
+		if (obuf)
+			return fileno(obuf->fp);
+		break;
+	}
+	return -1;
+}
+
 /*  do_select --- I/O multiplexing */
 
 static awk_value_t *
@@ -362,9 +382,11 @@ do_select(int nargs, awk_value_t *result)
 				if (((EL.value.val_type == AWK_UNDEFINED) || ((EL.value.val_type == AWK_STRING) && ! EL.value.str_value.len)) && (integer_string(EL.index.str_value.str, &x) == 0) && (x >= 0))
 					fds[i].array2fd[j] = x;
 				else if (EL.value.val_type == AWK_STRING) {
-					const awk_input_buf_t *buf;
-					if ((buf = get_file(EL.index.str_value.str, EL.index.str_value.len, EL.value.str_value.str, EL.value.str_value.len)) != NULL)
-						fds[i].array2fd[j] = buf->fd;
+					const awk_input_buf_t *ibuf;
+					const awk_output_buf_t *obuf;
+					int fd;
+					if (get_file(EL.index.str_value.str, EL.index.str_value.len, EL.value.str_value.str, EL.value.str_value.len, -1, &ibuf, &obuf) && ((fd = grabfd(i, ibuf, obuf, EL.index.str_value.str, EL.value.str_value.str)) >= 0))
+						fds[i].array2fd[j] = fd;
 					else
 						warning(ext_id, _("select: get_file(`%s', `%s') failed in `%s' array"), EL.index.str_value.str, EL.value.str_value.str, argname[i]);
 				}
@@ -566,11 +588,12 @@ do_set_non_blocking(int nargs, awk_value_t *result)
 	else if (get_argument(0, AWK_STRING, & cmd) &&
 		(get_argument(1, AWK_STRING, & cmdtype) ||
 			(! cmd.str_value.len && (nargs == 1)))) {
-		const awk_input_buf_t *buf;
-		if ((buf = get_file(cmd.str_value.str, cmd.str_value.len, cmdtype.str_value.str, cmdtype.str_value.len)) != NULL) {
-			int rc = set_non_blocking(buf->fd);
-			if (rc == 0)
-				set_retry(buf->name);
+		const awk_input_buf_t *ibuf;
+		const awk_output_buf_t *obuf;
+		if (get_file(cmd.str_value.str, cmd.str_value.len, cmdtype.str_value.str, cmdtype.str_value.len, -1, &ibuf, &obuf)) {
+			int rc = set_non_blocking(ibuf ? ibuf->fd : fileno(obuf->fp));
+			if (rc == 0 && ibuf)
+				set_retry(ibuf->name);
 			return make_number(rc, result);
 		}
 		warning(ext_id, _("set_non_blocking: get_file(`%s', `%s') failed"), cmd.str_value.str, cmdtype.str_value.str);
diff --git a/extension/testext.c b/extension/testext.c
index 7462265b..3ac1f124 100644
--- a/extension/testext.c
+++ b/extension/testext.c
@@ -37,6 +37,7 @@
 
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <fcntl.h>
 
 #include "gawkapi.h"
 
@@ -710,6 +711,7 @@ BEGIN {
 	ret = test_indirect_vars()	# should get correct value of NR
 	printf("test_indirect_var() return %d\n", ret)
 	delete ARGV[1]
+	print ""
 }
 */
 
@@ -742,6 +744,63 @@ out:
 	return result;
 }
 
+/*
+BEGIN {
+	outfile = "testexttmp.txt"
+	alias = ".test.alias"
+	print "line 1" > outfile
+	print "line 2" > outfile
+	print "line 3" > outfile
+	close(outfile)
+	ret = test_get_file(outfile, alias)
+	printf "test_get_file returned %d\n", ret
+	nr = 0
+	while ((getline < alias) > 0)
+		printf "File [%s] nr [%s]: %s\n", alias, ++nr, $0
+	close(alias)
+	system("rm " outfile)
+	print ""
+}
+*/
+
+/* test_get_file --- test that we can create a file */
+
+static awk_value_t *
+test_get_file(int nargs, awk_value_t *result)
+{
+	awk_value_t filename, alias;
+	int fd;
+	const awk_input_buf_t *ibuf;
+	const awk_output_buf_t *obuf;
+
+	if (nargs != 2) {
+		printf("%s: nargs not right (%d should be 2)\n", __func__, nargs);
+		return make_number(-1.0, result);
+	}
+
+	if (! get_argument(0, AWK_STRING, & filename)) {
+		printf("%s: cannot get first arg\n", __func__);
+		return make_number(-1.0, result);
+	}
+	if (! get_argument(1, AWK_STRING, & alias)) {
+		printf("%s: cannot get first arg\n", __func__);
+		return make_number(-1.0, result);
+	}
+	if ((fd = open(filename.str_value.str, O_RDONLY)) < 0) {
+		printf("%s: open(%s) failed\n", __func__, filename.str_value.str);
+		return make_number(-1.0, result);
+	}
+	if (! get_file(alias.str_value.str, strlen(alias.str_value.str), "<", 1, fd, &ibuf, &obuf)) {
+		printf("%s: get_file(%s) failed\n", __func__, alias.str_value.str);
+		return make_number(-1.0, result);
+	}
+	if (! ibuf || ibuf->fd != fd) {
+		printf("%s: get_file(%s) returned fd %d instead of %d\n", __func__, alias.str_value.str, ibuf ? ibuf->fd : -1, fd);
+		return make_number(-1.0, result);
+	}
+	return make_number(0.0, result);
+}
+
 /* fill_in_array --- fill in a new array */
 
 static void
@@ -837,6 +896,7 @@ static awk_ext_func_t func_table[] = {
 	{ "test_scalar", test_scalar, 1 },
 	{ "test_scalar_reserved", test_scalar_reserved, 0 },
 	{ "test_indirect_vars", test_indirect_vars, 0 },
+	{ "test_get_file", test_get_file, 2 },
 };
 
 /* init_testext --- additional initialization function */
diff --git a/gawkapi.c b/gawkapi.c
index 46aef7b6..00a101ea 100644
--- a/gawkapi.c
+++ b/gawkapi.c
@@ -1039,8 +1039,8 @@ api_release_value(awk_ext_id_t id, awk_value_cookie_t value)
 
 /* api_get_file --- return a handle to an existing or newly opened file */
 
-static const awk_input_buf_t *
-api_get_file(awk_ext_id_t id, const char *name, size_t namelen, const char *filetype, size_t typelen)
+static awk_bool_t
+api_get_file(awk_ext_id_t id, const char *name, size_t namelen, const char *filetype, size_t typelen, int fd, const awk_input_buf_t **ibufp, const awk_output_buf_t **obufp)
 {
 	const struct redirect *f;
 	int flag;	/* not used, sigh */
@@ -1049,7 +1049,7 @@ api_get_file(awk_ext_id_t id, const char *name, size_t namelen, const char *file
 	if ((name == NULL) || (namelen == 0)) {
 		if (curfile == NULL) {
 			if (nextfile(& curfile, false) <= 0)
-				return NULL;
+				return awk_false;
 			{
 				INSTRUCTION *pc = main_beginfile;
 				/* save execution state */
@@ -1072,7 +1072,9 @@ api_get_file(awk_ext_id_t id, const char *name, size_t namelen, const char *file
 				source = save_source;
 			}
 		}
-		return &curfile->public;
+		*ibufp = &curfile->public;
+		*obufp = NULL;
+		return awk_true;
 	}
 	redirtype = redirect_none;
 	switch (typelen) {
@@ -1110,11 +1112,13 @@ api_get_file(awk_ext_id_t id, const char *name, size_t namelen, const char *file
 	if (redirtype == redirect_none) {
 		warning(_("cannot open unrecognized file type `%s' for `%s'"),
 			filetype, name);
-		return NULL;
+		return awk_false;
 	}
-	if ((f = redirect_string(name, namelen, 0, redirtype, &flag)) == NULL)
-		return NULL;
-	return &f->iop->public;
+	if ((f = redirect_string(name, namelen, 0, redirtype, &flag, fd)) == NULL)
+		return awk_false;
+	*ibufp = f->iop ? & f->iop->public : NULL;
+	*obufp = f->output.fp ? & f->output : NULL;
+	return awk_true;
 }
 
 /*
diff --git a/gawkapi.h b/gawkapi.h
index dbe7fdf8..65f1dfc3 100644
--- a/gawkapi.h
+++ b/gawkapi.h
@@ -679,16 +679,34 @@ typedef struct gawk_api {
 	 * Look up a file.  If the name is NULL or name_len is 0, it returns
 	 * data for the currently open input file corresponding to FILENAME
 	 * (and it will not access the filetype or typelen arguments, so those
-	 * may be undefined).
+	 * may be undefined).  
 	 * If the file is not already open, it tries to open it.
 	 * The "filetype" argument should be one of:
 	 *    ">", ">>", "<", "|>", "|<", and "|&"
+	 * If the file is not already open, and the fd argument is non-negative,
+	 * gawk will use that file descriptor instead of opening the file
+	 * in the usual way.  If the fd is non-negative, but the file exists
+	 * already, gawk ignores the fd and returns the existing file.  It is
+	 * the caller's responsibility to notice that the fd in the returned
+	 * awk_input_buf_t does not match the requested value.  Note that
+	 * supplying a file descriptor is currently NOT supported for pipes.
+	 * It should work for input, output, append, and two-way (coprocess)
+	 * sockets.  If the filetype is two-way, we assume that it is a socket!
+	 * Note that in the two-way case, the intput and output file descriptors
+	 * may differ.  To check for success, one must check that either of
+	 * them matches.
 	 */
-	const awk_input_buf_t *(*api_get_file)(awk_ext_id_t id,
-						const char *name,
-						size_t name_len,
-						const char *filetype,
-						size_t typelen);
+	awk_bool_t (*api_get_file)(awk_ext_id_t id,
+			const char *name,
+			size_t name_len,
+			const char *filetype,
+			size_t typelen, int fd,
+			/*
+			 * Return values (on success, one or both should
+			 * be non-NULL):
+			 */
+			const awk_input_buf_t **ibufp,
+			const awk_output_buf_t **obufp);
 } gawk_api_t;
 
 #ifndef GAWK	/* these are not for the gawk code itself! */
@@ -771,8 +789,8 @@ typedef struct gawk_api {
 #define release_value(value) \
 	(api->api_release_value(ext_id, value))
 
-#define get_file(name, namelen, filetype, typelen) \
-	(api->api_get_file(ext_id, name, namelen, filetype, typelen))
+#define get_file(name, namelen, filetype, typelen, fd, ibuf, obuf) \
+	(api->api_get_file(ext_id, name, namelen, filetype, typelen, fd, ibuf, obuf))
 
 #define register_ext_version(version) \
 	(api->api_register_ext_version(ext_id, version))
diff --git a/io.c b/io.c
index 6d816da7..e9947da3 100644
--- a/io.c
+++ b/io.c
@@ -264,7 +264,7 @@ static IOBUF *iop_alloc(int fd, const char *name, int errno_val);
 static IOBUF *iop_finish(IOBUF *iop);
 static int gawk_pclose(struct redirect *rp);
 static int str2mode(const char *mode);
-static int two_way_open(const char *str, struct redirect *rp);
+static int two_way_open(const char *str, struct redirect *rp, int extfd);
 static int pty_vs_pipe(const char *command);
 static void find_input_parser(IOBUF *iop);
 static bool find_output_wrapper(awk_output_buf_t *outbuf);
@@ -720,7 +720,7 @@ redflags2str(int flags)
 /* redirect --- Redirection for printf and print commands */
 
 struct redirect *
-redirect_string(char *str, size_t explen, int not_string, int redirtype, int *errflg)
+redirect_string(const char *str, size_t explen, int not_string, int redirtype, int *errflg, int extfd)
 {
 	struct redirect *rp;
 	int tflag = 0;
@@ -848,7 +848,7 @@ redirect_string(char *str, size_t explen, int not_string, int redirtype, int *er
 		memcpy(newstr, str, explen);
 		newstr[explen] = '\0';
 		str = newstr;
-		rp->value = str;
+		rp->value = newstr;
 		rp->flag = tflag;
 		init_output_wrapper(& rp->output);
 		rp->output.name = str;
@@ -880,6 +880,10 @@ redirect_string(char *str, size_t explen, int not_string, int redirtype, int *er
 			mode = binmode("a");
 			break;
 		case redirect_pipe:
+			if (extfd >= 0) {
+				warning(_("get_file cannot create pipe `%s' with fd %d"), str, extfd);
+				return NULL;
+			}
 			/* synchronize output before new pipe */
 			(void) flush_io();
 
@@ -893,6 +897,10 @@ redirect_string(char *str, size_t explen, int not_string, int redirtype, int *er
 			rp->flag |= RED_NOBUF;
 			break;
 		case redirect_pipein:
+			if (extfd >= 0) {
+				warning(_("get_file cannot create pipe `%s' with fd %d"), str, extfd);
+				return NULL;
+			}
 			direction = "from";
 			if (gawk_popen(str, rp) == NULL)
 				fatal(_("can't open pipe `%s' for input (%s)"),
@@ -900,7 +908,7 @@ redirect_string(char *str, size_t explen, int not_string, int redirtype, int *er
 			break;
 		case redirect_input:
 			direction = "from";
-			fd = devopen(str, binmode("r"));
+			fd = (extfd >= 0) ? extfd : devopen(str, binmode("r"));
 			if (fd == INVALID_HANDLE && errno == EISDIR) {
 				*errflg = EISDIR;
 				/* do not free rp, saving it for reuse (save_rp = rp) */
@@ -917,8 +925,14 @@ redirect_string(char *str, size_t explen, int not_string, int redirtype, int *er
 			}
 			break;
 		case redirect_twoway:
+#ifndef HAVE_SOCKETS
+			if (extfd >= 0) {
+				warning(_("get_file socket creation not supported on this platform for `%s' with fd %d"), str, extfd);
+				return NULL;
+			}
+#endif
 			direction = "to/from";
-			if (! two_way_open(str, rp)) {
+			if (! two_way_open(str, rp, extfd)) {
 #ifdef HAVE_SOCKETS
 				if (inetfile(str, NULL)) {
 					*errflg = errno;
@@ -937,7 +951,7 @@ redirect_string(char *str, size_t explen, int not_string, int redirtype, int *er
 		if (mode != NULL) {
 			errno = 0;
 			rp->output.mode = mode;
-			fd = devopen(str, mode);
+			fd = (extfd >= 0) ? extfd : devopen(str, mode);
 
 			if (fd > INVALID_HANDLE) {
 				if (fd == fileno(stdin))
@@ -1042,7 +1056,7 @@ redirect(NODE *redir_exp, int redirtype, int *errflg)
 	int not_string = ((redir_exp->flags & STRCUR) == 0);
 	redir_exp = force_string(redir_exp);
 	return redirect_string(redir_exp->stptr, redir_exp->stlen, not_string,
-				redirtype, errflg);
+				redirtype, errflg, -1);
 }
 
 /* getredirect --- find the struct redirect for this file or pipe */
@@ -1700,16 +1714,16 @@ strictopen:
 /* two_way_open --- open a two way communications channel */
 
 static int
-two_way_open(const char *str, struct redirect *rp)
+two_way_open(const char *str, struct redirect *rp, int extfd)
 {
 	static bool no_ptys = false;
 
 #ifdef HAVE_SOCKETS
 	/* case 1: socket */
-	if (inetfile(str, NULL)) {
+	if (extfd >= 0 || inetfile(str, NULL)) {
 		int fd, newfd;
 
-		fd = devopen(str, "rw");
+		fd = (extfd >= 0) ? extfd : devopen(str, "rw");
 		if (fd == INVALID_HANDLE)
 			return false;
 		if ((BINMODE & BINMODE_OUTPUT) != 0)
diff --git a/test/ChangeLog b/test/ChangeLog
index 9d49a888..b6d60d97 100644
--- a/test/ChangeLog
+++ b/test/ChangeLog
@@ -1,3 +1,7 @@
+2014-11-06         Andrew J. Schorr     <aschorr@telemetry-investments.com>
+
+	* testext.ok: Add results from new test_get_file test.
+
 2014-11-02         Arnold D. Robbins     <arnold@skeeve.com>
 
 	* Makefile.am (profile7): New test.
diff --git a/test/testext.ok b/test/testext.ok
index 9b36bf72..5a78c159 100644
--- a/test/testext.ok
+++ b/test/testext.ok
@@ -69,6 +69,12 @@ test_scalar_reserved: could not update new_value2 for ARGC - pass
 test_indirect_var: sym_lookup of NR passed
 test_indirect_var: value of NR is 3
 test_indirect_var() return 1
+
+test_get_file returned 0
+File [.test.alias] nr [1]: line 1
+File [.test.alias] nr [2]: line 2
+File [.test.alias] nr [3]: line 3
+
 answer_num = 42
 message_string = hello, world
 new_array["hello"] = "world"