diff options
Diffstat (limited to 'extension')
| -rw-r--r-- | extension/ChangeLog | 71 | ||||
| -rw-r--r-- | extension/rwarray.3am | 35 | ||||
| -rw-r--r-- | extension/rwarray.c | 296 | ||||
| -rw-r--r-- | extension/testext.c | 77 |
4 files changed, 402 insertions, 77 deletions
diff --git a/extension/ChangeLog b/extension/ChangeLog index 5ce63c68..8872ec7b 100644 --- a/extension/ChangeLog +++ b/extension/ChangeLog @@ -2,6 +2,77 @@ * rwarray.c (write_number, read_number): Reformat comments a bit. +2021-12-09 Andrew J. Schorr <aschorr@telemetry-investments.com> + + * rwarray.c (write_number, read_number): Use + mpfr_get_default_rounding_mode() instead of arbitrarily choosing + MPFR_RNDN, taking advantage of the fact that core gawk maintains + this using the ROUNDMODE global variable. + +2021-12-08 Andrew J. Schorr <aschorr@telemetry-investments.com> + + * rwarray.c: Fix valgrind complaints related to creating mpz and mpfr + values on the stack in read_number by passing down storage from + the calling function that loads the data into gawk. + (value_storage): New union type to contain mpz_t or mpfr_t data. + (read_global): Allocate value_storage on the stack and pass a pointer + to read_elem. + (read_array): Ditto. + (read_elem): Receive new arg pointing to value_storage, and pass it + down to read_value. + (read_value): Receive new arg pointing to value_storage, and pass it + down to read_number. + (read_number): Receive new arg pointing to value_storage, and create + mpz and mpfr variables using that storage instead of in the local + scope. + +2021-12-08 Andrew J. Schorr <aschorr@telemetry-investments.com> + + * rwarray.c: Add new functions writeall and readall to implement + persistent state. + (write_backend): New helper function containing most of the logic + from do_writea. Note that we do not need to check nargs < 2 because + gawk will issue a fatal error if a function is called with fewer + than min_required_args. Clean up some minor issues with error + handling. + (do_writea): Grab the array argument and use write_backend to + do the rest of the work. + (do_writeall): Lookup SYMTAB and invoke write_backend. + (free_value): New function to free memory for data we end up ignoring + because the variables exist already. + (do_poke): Attempt to create variables that don't exist already or + are undefined. + (regular_array_handle): Wrapper around create_array. + (global_array_handle): Call create_array unless the variable exists + already and is an array with zero elements. + (read_global): New function used by readall to load global variables + from a file. + (read_one): New function to read a single array from a file. + (read_backend): New helper function containing most of the logic + from do_reada. Remove the superfluous nargs check. Read the file + prologue and then call read_global or read_one as appropriate to load + the data. + (do_reada): Grab the array argument and call read_backend with + read_one to load the data. + (do_readall): Call read_backend with read_global to load the data. + (read_array): Call read_elem with additional arg regular_array_handle. + (read_elem): Add a function argument controlling array creation to + pass down to read_value. + (read_value): Add a function argument to call for array creation + instead of calling create_array directly, since we may need to use + an existing array when populating global arrays in readall. + (func_table): Add writeall and readall. + * rwarray.3am: Document new functions writeall and readall. + +2021-12-08 Andrew J. Schorr <aschorr@telemetry-investments.com> + + * testext.c (test_array_create): New function to create an array + by name that enables testing whether an undefined variable can + be converted by the API into an array. + (populate_array): New helper function. + (fill_in_array): Use populate_array to fill in the elements. + (func_table): Add test_array_create. + 2021-12-08 Andrew J. Schorr <aschorr@telemetry-investments.com> * rwarray.c (write_number): Since mpfr_fpif_export is experimental diff --git a/extension/rwarray.3am b/extension/rwarray.3am index f17ffaa9..b10545a3 100644 --- a/extension/rwarray.3am +++ b/extension/rwarray.3am @@ -1,6 +1,6 @@ .TH RWARRAY 3am "Feb 02 2018" "Free Software Foundation" "GNU Awk Extension Modules" .SH NAME -writea, reada \- write and read gawk arrays to/from files +writea, reada, writeall, readall \- write and read gawk arrays to/from files .SH SYNOPSIS .ft CW @load "rwarray" @@ -8,14 +8,20 @@ writea, reada \- write and read gawk arrays to/from files ret = writea(file, array) .br ret = reada(file, array) +.br +ret = writeall(file) +.br +ret = readall(file) .ft R .SH DESCRIPTION The .I rwarray -extension adds two functions named -.BR writea() . -and +extension adds functions named +.BR writea() , .BR reada() , +.BR writeaall() , +and +.BR readaall() , as follows. .TP .B writea() @@ -33,6 +39,23 @@ it reads the file named as its first argument, filling in the array named as the second argument. It clears the array first. Here too, the return value is one on success and zero upon failure. +.TP +.B writeall() +This function takes a string argument, which is the name of the +file to which dump the state of all variables. Calling this function +is completely equivalent to calling +.B writea() +with the second argument equal to +.BR SYMTAB . +It returns one on success, or zero upon failure. +.TP +.B readall() +This function takes a string argument, which is the name of the +file from which to read the contents of various global variables. +For each variable in the file, the data is loaded unless the variable +already exists. If the variable already exists, the data for that variable +in the file is ignored. +It returns one on success, or zero upon failure. .SH NOTES The array created by .B reada() @@ -62,6 +85,10 @@ restored on systems with a different one, but this has not been tried. ret = writea("arraydump.bin", array) \&... ret = reada("arraydump.bin", array) +\&... +ret = writeall("globalstate.bin") +\&... +ret = readall("globalstate.bin") .fi .ft R .SH "SEE ALSO" diff --git a/extension/rwarray.c b/extension/rwarray.c index 532a8da1..af6f1cda 100644 --- a/extension/rwarray.c +++ b/extension/rwarray.c @@ -78,10 +78,16 @@ static awk_bool_t write_elem(FILE *fp, awk_element_t *element); static awk_bool_t write_value(FILE *fp, awk_value_t *val); static awk_bool_t write_number(FILE *fp, awk_value_t *val); +typedef union { + mpz_t mpz_val; + mpfr_t mpfr_val; +} value_storage; + +typedef awk_array_t (*array_handle_t)(awk_value_t *); static awk_bool_t read_array(FILE *fp, awk_array_t array); -static awk_bool_t read_elem(FILE *fp, awk_element_t *element); -static awk_bool_t read_value(FILE *fp, awk_value_t *value); -static awk_bool_t read_number(FILE *fp, awk_value_t *value, uint32_t code); +static awk_bool_t read_elem(FILE *fp, awk_element_t *element, array_handle_t, value_storage *); +static awk_bool_t read_value(FILE *fp, awk_value_t *value, array_handle_t, awk_value_t *idx, value_storage *vs); +static awk_bool_t read_number(FILE *fp, awk_value_t *value, uint32_t code, value_storage *); /* * Format of array info: @@ -117,12 +123,12 @@ static awk_bool_t read_number(FILE *fp, awk_value_t *value, uint32_t code); #define VT_BOOL 8 #define VT_UNDEFINED 20 -/* do_writea --- write an array */ +/* write_backend --- write an array */ static awk_value_t * -do_writea(int nargs, awk_value_t *result, struct awk_ext_func *unused) +write_backend(awk_value_t *result, awk_array_t array, const char *name) { - awk_value_t filename, array; + awk_value_t filename; FILE *fp = NULL; uint32_t major = MAJOR; uint32_t minor = MINOR; @@ -130,18 +136,9 @@ do_writea(int nargs, awk_value_t *result, struct awk_ext_func *unused) assert(result != NULL); make_number(0.0, result); - if (nargs < 2) - goto out; - - /* filename is first arg, array to dump is second */ + /* filename is first arg */ if (! get_argument(0, AWK_STRING, & filename)) { - warning(ext_id, _("do_writea: first argument is not a string")); - errno = EINVAL; - goto done1; - } - - if (! get_argument(1, AWK_ARRAY, & array)) { - warning(ext_id, _("do_writea: second argument is not an array")); + warning(ext_id, _("%s: first argument is not a string"), name); errno = EINVAL; goto done1; } @@ -162,21 +159,55 @@ do_writea(int nargs, awk_value_t *result, struct awk_ext_func *unused) if (fwrite(& minor, 1, sizeof(minor), fp) != sizeof(minor)) goto done1; - if (write_array(fp, array.array_cookie)) { + if (write_array(fp, array)) { make_number(1.0, result); - goto done0; + fclose(fp); + return result; } done1: update_ERRNO_int(errno); - unlink(filename.str_value.str); - -done0: - fclose(fp); -out: + if (fp != NULL) { + fclose(fp); + unlink(filename.str_value.str); + } return result; } +/* do_writea --- write an array */ + +static awk_value_t * +do_writea(int nargs, awk_value_t *result, struct awk_ext_func *unused) +{ + awk_value_t array; + + if (! get_argument(1, AWK_ARRAY, & array)) { + warning(ext_id, _("writea: second argument is not an array")); + errno = EINVAL; + update_ERRNO_int(errno); + make_number(0.0, result); + return result; + } + return write_backend(result, array.array_cookie, "writea"); +} + +/* do_writeall --- write out SYMTAB */ + +static awk_value_t * +do_writeall(int nargs, awk_value_t *result, struct awk_ext_func *unused) +{ + awk_value_t array; + + if (! sym_lookup("SYMTAB", AWK_ARRAY, & array)) { + warning(ext_id, _("writeall: unable to find SYMTAB array")); + errno = EINVAL; + update_ERRNO_int(errno); + make_number(0.0, result); + return result; + } + return write_backend(result, array.array_cookie, "writeall"); +} + /* write_array --- write out an array or a sub-array */ @@ -340,7 +371,7 @@ write_number(FILE *fp, awk_value_t *val) if (mpfr_fpif_export(fp, val->num_ptr) != 0) #else #define MPFR_STR_BASE 62 /* maximize base to minimize string len */ -#define MPFR_STR_ROUND MPFR_RNDN +#define MPFR_STR_ROUND mpfr_get_default_rounding_mode() /* * XXX does the choice of MPFR_RNDN matter, given * that the precision is 0, so we should be rendering @@ -367,12 +398,139 @@ write_number(FILE *fp, awk_value_t *val) return awk_true; } -/* do_reada --- read an array */ +/* free_value --- release memory for ignored global variables */ + +static void +free_value(awk_value_t *v) +{ + switch (v->val_type) { + case AWK_ARRAY: + clear_array(v->array_cookie); + break; + case AWK_STRING: + case AWK_REGEX: + case AWK_STRNUM: + case AWK_UNDEFINED: + gawk_free(v->str_value.str); + break; + case AWK_BOOL: + /* no memory allocated */ + break; + case AWK_NUMBER: + switch (v->num_type) { + case AWK_NUMBER_TYPE_DOUBLE: + /* no memory allocated */ + break; + case AWK_NUMBER_TYPE_MPZ: + mpz_clear(v->num_ptr); + break; + case AWK_NUMBER_TYPE_MPFR: + mpfr_clear(v->num_ptr); + break; + default: + warning(ext_id, _("cannot free number with unknown type %d"), v->num_type); + break; + } + break; + default: + warning(ext_id, _("cannot free value with unhandled type %d"), v->val_type); + break; + } +} + +/* do_poke --- create a global variable */ + +static awk_bool_t +do_poke(awk_element_t *e) +{ + awk_value_t t; + + if (e->index.val_type != AWK_STRING) + return awk_false; + /* So this is a bit tricky. If the program refers to the variable, + * then it will already exist in an undefined state after parsing. + * If the program never refers to it, then the lookup fails. + * We still need to create it in case the program accesses it via + * indirection through the SYMTAB table. */ + if (sym_lookup(e->index.str_value.str, AWK_UNDEFINED, &t) && (t.val_type != AWK_UNDEFINED)) + return awk_false; + if (! sym_update(e->index.str_value.str, & e->value)) { + warning(ext_id, _("readall: unable to set %s"), e->index.str_value.str); + return awk_false; + } + return awk_true; +} + +/* regular_array_handle --- array creation hook for normal reada */ + +static awk_array_t +regular_array_handle(awk_value_t *unused) +{ + return create_array(); +} + +/* global_array_handle --- array creation hook for readall */ + +static awk_array_t +global_array_handle(awk_value_t *n) +{ + awk_value_t t; + size_t count; + + /* The array may exist already because it was instantiated during + * program parsing, so we use the existing array if it is empty. */ + return ((n->val_type == AWK_STRING) && sym_lookup(n->str_value.str, AWK_UNDEFINED, &t) && (t.val_type == AWK_ARRAY) && get_element_count(t.array_cookie, & count) && ! count) ? t.array_cookie : create_array(); +} + +/* read_global --- read top-level variables dumped from SYMTAB */ + +static awk_bool_t +read_global(FILE *fp, awk_array_t unused) +{ + uint32_t i; + uint32_t count; + awk_element_t new_elem; + value_storage vs; + + if (fread(& count, 1, sizeof(count), fp) != sizeof(count)) + return awk_false; + + count = ntohl(count); + + for (i = 0; i < count; i++) { + if (read_elem(fp, & new_elem, global_array_handle, &vs)) { + if (! do_poke(& new_elem)) + free_value(& new_elem.value); + if (new_elem.index.str_value.len) + /* free string allocated by make_const_string */ + gawk_free(new_elem.index.str_value.str); + } else + return awk_false; + } + + return awk_true; +} + +/* read_one --- read one array */ + +static awk_bool_t +read_one(FILE *fp, awk_array_t array) +{ + if (! clear_array(array)) { + errno = ENOMEM; + warning(ext_id, _("reada: clear_array failed")); + return awk_false; + } + + return read_array(fp, array); +} + +/* read_backend --- common code for reada and readall */ static awk_value_t * -do_reada(int nargs, awk_value_t *result, struct awk_ext_func *unused) +read_backend(awk_value_t *result, awk_array_t array, const char *name, awk_bool_t (*func)(FILE *, awk_array_t)) { - awk_value_t filename, array; + awk_value_t filename; FILE *fp = NULL; uint32_t major; uint32_t minor; @@ -381,18 +539,9 @@ do_reada(int nargs, awk_value_t *result, struct awk_ext_func *unused) assert(result != NULL); make_number(0.0, result); - if (nargs < 2) - goto out; - - /* directory is first arg, array to read is second */ + /* filename is first arg */ if (! get_argument(0, AWK_STRING, & filename)) { - warning(ext_id, _("do_reada: first argument is not a string")); - errno = EINVAL; - goto done1; - } - - if (! get_argument(1, AWK_ARRAY, & array)) { - warning(ext_id, _("do_reada: second argument is not an array")); + warning(ext_id, _("%s: first argument is not a string"), name); errno = EINVAL; goto done1; } @@ -434,13 +583,7 @@ do_reada(int nargs, awk_value_t *result, struct awk_ext_func *unused) goto done1; } - if (! clear_array(array.array_cookie)) { - errno = ENOMEM; - warning(ext_id, _("do_reada: clear_array failed")); - goto done1; - } - - if (read_array(fp, array.array_cookie)) { + if ((*func)(fp, array)) { make_number(1.0, result); goto done0; } @@ -450,10 +593,34 @@ done1: done0: if (fp != NULL) fclose(fp); -out: return result; } +/* do_reada --- read an array */ + +static awk_value_t * +do_reada(int nargs, awk_value_t *result, struct awk_ext_func *unused) +{ + awk_value_t array; + + if (! get_argument(1, AWK_ARRAY, & array)) { + warning(ext_id, _("reada: second argument is not an array")); + errno = EINVAL; + update_ERRNO_int(errno); + make_number(0.0, result); + return result; + } + return read_backend(result, array.array_cookie, "read", read_one); +} + +/* do_readall --- read top-level variables */ + +static awk_value_t * +do_readall(int nargs, awk_value_t *result, struct awk_ext_func *unused) +{ + return read_backend(result, NULL, "readall", read_global); +} + /* read_array --- read in an array or sub-array */ @@ -463,6 +630,7 @@ read_array(FILE *fp, awk_array_t array) uint32_t i; uint32_t count; awk_element_t new_elem; + value_storage vs; if (fread(& count, 1, sizeof(count), fp) != sizeof(count)) return awk_false; @@ -470,7 +638,7 @@ read_array(FILE *fp, awk_array_t array) count = ntohl(count); for (i = 0; i < count; i++) { - if (read_elem(fp, & new_elem)) { + if (read_elem(fp, & new_elem, regular_array_handle, &vs)) { /* add to array */ if (! set_array_element_by_elem(array, & new_elem)) { warning(ext_id, _("read_array: set_array_element failed")); @@ -489,7 +657,7 @@ read_array(FILE *fp, awk_array_t array) /* read_elem --- read in a single element */ static awk_bool_t -read_elem(FILE *fp, awk_element_t *element) +read_elem(FILE *fp, awk_element_t *element, array_handle_t array_handle, value_storage *vs) { uint32_t index_len; static char *buffer; @@ -527,7 +695,7 @@ read_elem(FILE *fp, awk_element_t *element) make_null_string(& element->index); } - if (! read_value(fp, & element->value)) + if (! read_value(fp, & element->value, array_handle, & element->index, vs)) return awk_false; return awk_true; @@ -536,7 +704,7 @@ read_elem(FILE *fp, awk_element_t *element) /* read_value --- read a number or a string */ static awk_bool_t -read_value(FILE *fp, awk_value_t *value) +read_value(FILE *fp, awk_value_t *value, array_handle_t array_handle, awk_value_t *idx, value_storage *vs) { uint32_t code, len; @@ -546,7 +714,7 @@ read_value(FILE *fp, awk_value_t *value) code = ntohl(code); if (code == VT_ARRAY) { - awk_array_t array = create_array(); + awk_array_t array = (*array_handle)(idx); if (! read_array(fp, array)) return awk_false; @@ -557,7 +725,7 @@ read_value(FILE *fp, awk_value_t *value) } else if (code == VT_NUMBER || code == VT_GMP || code == VT_MPFR) { - return read_number(fp, value, code); + return read_number(fp, value, code, vs); } else { if (fread(& len, 1, sizeof(len), fp) != sizeof(len)) { return awk_false; @@ -610,7 +778,7 @@ read_value(FILE *fp, awk_value_t *value) /* read_number --- read a double, GMP, or MPFR number */ static awk_bool_t -read_number(FILE *fp, awk_value_t *value, uint32_t code) +read_number(FILE *fp, awk_value_t *value, uint32_t code, value_storage *vs) { uint32_t len; @@ -632,28 +800,24 @@ read_number(FILE *fp, awk_value_t *value, uint32_t code) } else { #ifdef HAVE_MPFR if (code == VT_GMP) { - mpz_t mp_ptr; - - mpz_init(mp_ptr); - if (mpz_inp_raw(mp_ptr, fp) == 0) + mpz_init(vs->mpz_val); + if (mpz_inp_raw(vs->mpz_val, fp) == 0) return awk_false; - value = make_number_mpz(mp_ptr, value); + value = make_number_mpz(vs->mpz_val, value); } else { - mpfr_t mpfr_val; - mpfr_init(mpfr_val); - + mpfr_init(vs->mpfr_val); #ifdef USE_MPFR_FPIF /* preferable if widely available and stable */ - if (mpfr_fpif_import(mpfr_val, fp) != 0) + if (mpfr_fpif_import(vs->mpfr_val, fp) != 0) #else // N.B. need to consume the terminating space we wrote // after mpfr_out_str - if ((mpfr_inp_str(mpfr_val, fp, MPFR_STR_BASE, MPFR_STR_ROUND) == 0) || (getc(fp) != ' ')) + if ((mpfr_inp_str(vs->mpfr_val, fp, MPFR_STR_BASE, MPFR_STR_ROUND) == 0) || (getc(fp) != ' ')) #endif return awk_false; - value = make_number_mpfr(& mpfr_val, value); + value = make_number_mpfr(vs->mpfr_val, value); } #else fatal(ext_id(_("rwarray extension: GMP/MPFR value in file but compiled without GMP/MPFR support.")); @@ -666,6 +830,8 @@ read_number(FILE *fp, awk_value_t *value, uint32_t code) static awk_ext_func_t func_table[] = { { "writea", do_writea, 2, 2, awk_false, NULL }, { "reada", do_reada, 2, 2, awk_false, NULL }, + { "writeall", do_writeall, 1, 1, awk_false, NULL }, + { "readall", do_readall, 1, 1, awk_false, NULL }, }; diff --git a/extension/testext.c b/extension/testext.c index bfaa8637..18465f2a 100644 --- a/extension/testext.c +++ b/extension/testext.c @@ -48,6 +48,7 @@ static const char *ext_version = "testext extension: version 1.0"; int plugin_is_GPL_compatible; static void fill_in_array(awk_value_t *value); +static int populate_array(awk_array_t); #ifdef __MINGW32__ unsigned int @@ -666,6 +667,54 @@ out: } /* +function tfunc(f) { + if (isarray(f)) + print "good: we have an array" +} + +BEGIN { + printf "test_array_create returned %d\n", test_array_create("testarr") + tfunc(testarr) +} +*/ + +static awk_value_t * +test_array_create(int nargs, awk_value_t *result, struct awk_ext_func *unused) +{ + awk_value_t new_array; + awk_value_t arg0; + + (void) nargs; /* silence warnings */ + make_number(0.0, result); + + if (! get_argument(0, AWK_STRING, & arg0)) { + printf("test_array_create: could not get argument\n"); + goto out; + } + + if (arg0.val_type != AWK_STRING) { + printf("test_array_create: argument is not string (%d)\n", + arg0.val_type); + goto out; + } + + new_array.val_type = AWK_ARRAY; + new_array.array_cookie = create_array(); + if (! sym_update(arg0.str_value.str, & new_array)) { + printf("test_array_create: sym_update(\"%s\") failed!\n", arg0.str_value.str); + goto out; + } + if (populate_array(new_array.array_cookie) < 0) { + printf("test_array_create: populate(\"%s\") failed!\n", arg0.str_value.str); + goto out; + } + + make_number(1.0, result); +out: + return result; +} + +/* BEGIN { printf("Initial value of LINT is %d\n", LINT) ret = print_do_lint(); @@ -958,29 +1007,40 @@ do_get_file(int nargs, awk_value_t *result, struct awk_ext_func *unused) return make_number(1.0, result); } -/* fill_in_array --- fill in a new array */ +/* populate_array --- fill in some array values */ -static void -fill_in_array(awk_value_t *new_array) +static int +populate_array(awk_array_t a_cookie) { - awk_array_t a_cookie; awk_value_t index, value; - a_cookie = create_array(); - (void) make_const_string("hello", 5, & index); (void) make_const_string("world", 5, & value); if (! set_array_element(a_cookie, & index, & value)) { printf("fill_in_array:%d: set_array_element failed\n", __LINE__); - return; + return -1; } (void) make_const_string("answer", 6, & index); (void) make_number(42.0, & value); if (! set_array_element(a_cookie, & index, & value)) { printf("fill_in_array:%d: set_array_element failed\n", __LINE__); - return; + return -1; } + return 0; +} + +/* fill_in_array --- fill in a new array */ + +static void +fill_in_array(awk_value_t *new_array) +{ + awk_array_t a_cookie; + + a_cookie = create_array(); + + if (populate_array(a_cookie) < 0) + return; new_array->val_type = AWK_ARRAY; new_array->array_cookie = a_cookie; @@ -1061,6 +1121,7 @@ static awk_ext_func_t func_table[] = { { "test_array_size", test_array_size, 1, 1, awk_false, NULL }, { "test_array_elem", test_array_elem, 2, 2, awk_false, NULL }, { "test_array_param", test_array_param, 1, 1, awk_false, NULL }, + { "test_array_create", test_array_create, 1, 1, awk_false, NULL }, { "print_do_lint", print_do_lint, 0, 0, awk_false, NULL }, { "test_scalar", test_scalar, 1, 1, awk_false, NULL }, { "test_scalar_reserved", test_scalar_reserved, 0, 0, awk_false, NULL }, |