From 7c03ff1f225dc9cfe857621ee52a066d7d3d0123 Mon Sep 17 00:00:00 2001 From: Israel Barth Date: Fri, 15 Jul 2022 20:56:36 -0300 Subject: [PATCH v1] Added support for DEFAULT in COPY FROM Previous to this commit, COPY FROM command used to load the column DEFAULT value if the column was missing in the command specification. With this commit we introduce a new feature that works like the NULL feature from COPY command. The user will be able to specify a marker, and whenever that marker is found in the input of COPY FROM, it will be replaced with the DEFAULT value of the corresponding column. In order to make "the new" COPY from backward compatible, the DEFAULT feature will only take place if the use specifies some value for that option. We are taking advantage of the code that was already implemented in COPY FROM to find and evaluate the DEFAULT expressions, with the only difference that now we check both for columns that are missing in the command specification, and for columns where the DEFAULT marker was found. Signed-off-by: Israel Barth --- doc/src/sgml/ref/copy.sgml | 13 +++ src/backend/commands/copy.c | 46 ++++++++++ src/backend/commands/copyfrom.c | 22 +++-- src/backend/commands/copyfromparse.c | 100 ++++++++++++++++++---- src/include/commands/copy.h | 4 +- src/include/commands/copyfrom_internal.h | 6 +- src/test/regress/expected/copydefault.out | 76 ++++++++++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/copydefault.sql | 78 +++++++++++++++++ 9 files changed, 319 insertions(+), 28 deletions(-) create mode 100644 src/test/regress/expected/copydefault.out create mode 100644 src/test/regress/sql/copydefault.sql diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 63afa0d97e..4cac339b34 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -43,6 +43,7 @@ COPY { table_name [ ( column_name [, ...] ) FORCE_NULL ( column_name [, ...] ) ENCODING 'encoding_name' + DEFAULT 'default_string' @@ -368,6 +369,18 @@ COPY { table_name [ ( + + DEFAULT + + + Specifies the string that represents a default value. Each time this string + is found in the input file, the default value of the corresponding column + will be used. + This option is not allowed when using binary format. + + + + WHERE diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 49924e476a..2d59556c4b 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -460,6 +460,12 @@ ProcessCopyOptions(ParseState *pstate, errorConflictingDefElem(defel, pstate); opts_out->null_print = defGetString(defel); } + else if (strcmp(defel->defname, "default") == 0) + { + if (opts_out->default_print) + errorConflictingDefElem(defel, pstate); + opts_out->default_print = defGetString(defel); + } else if (strcmp(defel->defname, "header") == 0) { if (header_specified) @@ -573,6 +579,11 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot specify NULL in BINARY mode"))); + if (opts_out->binary && opts_out->default_print) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot specify DEFAULT in BINARY mode"))); + /* Set defaults for omitted options */ if (!opts_out->delim) opts_out->delim = opts_out->csv_mode ? "," : "\t"; @@ -608,6 +619,17 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY null representation cannot use newline or carriage return"))); + if (opts_out->default_print) + { + opts_out->default_print_len = strlen(opts_out->default_print); + + if (strchr(opts_out->default_print, '\r') != NULL || + strchr(opts_out->default_print, '\n') != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY default representation cannot use newline or carriage return"))); + } + /* * Disallow unsafe delimiter characters in non-CSV mode. We can't allow * backslash because it would be ambiguous. We can't allow the other @@ -701,6 +723,30 @@ ProcessCopyOptions(ParseState *pstate, ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("CSV quote character must not appear in the NULL specification"))); + + if (opts_out->default_print) + { + /* Don't allow the delimiter to appear in the default string. */ + if (strchr(opts_out->default_print, opts_out->delim[0]) != NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY delimiter must not appear in the DEFAULT specification"))); + + /* Don't allow the CSV quote char to appear in the default string. */ + if (opts_out->csv_mode && + strchr(opts_out->default_print, opts_out->quote[0]) != NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("CSV quote character must not appear in the DEFAULT specification"))); + + /* Don't allow the NULL and DEFAULT string to be the same */ + if (opts_out->null_print_len == opts_out->default_print_len && + strncmp(opts_out->null_print, opts_out->default_print, + opts_out->null_print_len) == 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("NULL specification and DEFAULT specification cannot be the same"))); + } } /* diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index a976008b3d..1c430217bb 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1418,12 +1418,12 @@ BeginCopyFrom(ParseState *pstate, &in_func_oid, &typioparams[attnum - 1]); fmgr_info(in_func_oid, &in_functions[attnum - 1]); - /* Get default info if needed */ - if (!list_member_int(cstate->attnumlist, attnum) && !att->attgenerated) + /* Get default info if available */ + defexprs[attnum - 1] = NULL; + + if(!att->attgenerated) { - /* attribute is NOT to be copied from input */ - /* use default value if one exists */ - Expr *defexpr = (Expr *) build_column_default(cstate->rel, + Expr *defexpr = (Expr *) build_column_default(cstate->rel, attnum); if (defexpr != NULL) @@ -1432,9 +1432,15 @@ BeginCopyFrom(ParseState *pstate, defexpr = expression_planner(defexpr); /* Initialize executable expression in copycontext */ - defexprs[num_defaults] = ExecInitExpr(defexpr, NULL); - defmap[num_defaults] = attnum - 1; - num_defaults++; + defexprs[attnum - 1] = ExecInitExpr(defexpr, NULL); + + /* attribute is NOT to be copied from input */ + /* use default value if one exists */ + if (!list_member_int(cstate->attnumlist, attnum)) + { + defmap[num_defaults] = attnum - 1; + num_defaults++; + } /* * If a default expression looks at the table being loaded, diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 57813b3458..0f8d79a30b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -151,8 +151,8 @@ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"; /* non-export function prototypes */ static bool CopyReadLine(CopyFromState cstate); static bool CopyReadLineText(CopyFromState cstate); -static int CopyReadAttributesText(CopyFromState cstate); -static int CopyReadAttributesCSV(CopyFromState cstate); +static int CopyReadAttributesText(CopyFromState cstate, bool *defaults); +static int CopyReadAttributesCSV(CopyFromState cstate, bool *defaults); static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull); @@ -751,7 +751,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) * NOTE: force_not_null option are not applied to the returned fields. */ bool -NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) +NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields, bool *defaults) { int fldct; bool done; @@ -775,9 +775,9 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) int fldnum; if (cstate->opts.csv_mode) - fldct = CopyReadAttributesCSV(cstate); + fldct = CopyReadAttributesCSV(cstate, defaults); else - fldct = CopyReadAttributesText(cstate); + fldct = CopyReadAttributesText(cstate, defaults); if (fldct != list_length(cstate->attnumlist)) ereport(ERROR, @@ -830,9 +830,9 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) /* Parse the line into de-escaped field values */ if (cstate->opts.csv_mode) - fldct = CopyReadAttributesCSV(cstate); + fldct = CopyReadAttributesCSV(cstate, defaults); else - fldct = CopyReadAttributesText(cstate); + fldct = CopyReadAttributesText(cstate, defaults); *fields = cstate->raw_fields; *nfields = fldct; @@ -862,6 +862,7 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, int i; int *defmap = cstate->defmap; ExprState **defexprs = cstate->defexprs; + bool *defaults; tupDesc = RelationGetDescr(cstate->rel); num_phys_attrs = tupDesc->natts; @@ -870,6 +871,8 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, /* Initialize all values for row to NULL */ MemSet(values, 0, num_phys_attrs * sizeof(Datum)); MemSet(nulls, true, num_phys_attrs * sizeof(bool)); + defaults = (bool *) palloc0(num_phys_attrs * sizeof(bool)); + MemSet(defaults, false, num_phys_attrs * sizeof(bool)); if (!cstate->opts.binary) { @@ -880,7 +883,7 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, char *string; /* read raw fields in the next line */ - if (!NextCopyFromRawFields(cstate, &field_strings, &fldct)) + if (!NextCopyFromRawFields(cstate, &field_strings, &fldct, defaults)) return false; /* check for overflowing fields */ @@ -938,12 +941,27 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, cstate->cur_attname = NameStr(att->attname); cstate->cur_attval = string; - values[m] = InputFunctionCall(&in_functions[m], - string, - typioparams[m], - att->atttypmod); + if (string != NULL) nulls[m] = false; + + if (defaults[m]) + { + /* + * The caller must supply econtext and have switched into the + * per-tuple memory context in it. + */ + Assert(econtext != NULL); + Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory); + + values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]); + } + else + values[m] = InputFunctionCall(&in_functions[m], + string, + typioparams[m], + att->atttypmod); + cstate->cur_attname = NULL; cstate->cur_attval = NULL; } @@ -1019,10 +1037,12 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, Assert(econtext != NULL); Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory); - values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext, + values[defmap[i]] = ExecEvalExpr(defexprs[defmap[i]], econtext, &nulls[defmap[i]]); } + pfree(defaults); + return true; } @@ -1475,7 +1495,7 @@ GetDecimalFromHex(char hex) * The return value is the number of fields actually read. */ static int -CopyReadAttributesText(CopyFromState cstate) +CopyReadAttributesText(CopyFromState cstate, bool *defaults) { char delimc = cstate->opts.delim[0]; int fieldno; @@ -1663,6 +1683,31 @@ CopyReadAttributesText(CopyFromState cstate) if (input_len == cstate->opts.null_print_len && strncmp(start_ptr, cstate->opts.null_print, input_len) == 0) cstate->raw_fields[fieldno] = NULL; + /* Check whether raw input matched default marker */ + else if (cstate->opts.default_print && + input_len == cstate->opts.default_print_len && + strncmp(start_ptr, cstate->opts.default_print, input_len) == 0) + { + /* fieldno is 0-index and attnum is 1-index */ + int m = list_nth_int(cstate->attnumlist, fieldno) - 1; + + if (cstate->defexprs[m] != NULL) + { + /* defaults contain entries for all physical attributes */ + defaults[m] = true; + } + else + { + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("unexpected DEFAULT in COPY data"), + errdetail("Column \"%s\" has no DEFAULT value.", + NameStr(att->attname)))); + } + } else { /* @@ -1703,7 +1748,7 @@ CopyReadAttributesText(CopyFromState cstate) * "standard" (i.e. common) CSV usage. */ static int -CopyReadAttributesCSV(CopyFromState cstate) +CopyReadAttributesCSV(CopyFromState cstate, bool *defaults) { char delimc = cstate->opts.delim[0]; char quotec = cstate->opts.quote[0]; @@ -1852,6 +1897,31 @@ endfield: if (!saw_quote && input_len == cstate->opts.null_print_len && strncmp(start_ptr, cstate->opts.null_print, input_len) == 0) cstate->raw_fields[fieldno] = NULL; + /* Check whether raw input matched default marker */ + else if (cstate->opts.default_print && + input_len == cstate->opts.default_print_len && + strncmp(start_ptr, cstate->opts.default_print, input_len) == 0) + { + /* fieldno is 0-index and attnum is 1-index */ + int m = list_nth_int(cstate->attnumlist, fieldno) - 1; + + if (cstate->defexprs[m] != NULL) + { + /* defaults contain entries for all physical attributes */ + defaults[m] = true; + } + else + { + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("unexpected DEFAULT in COPY data"), + errdetail("Column \"%s\" has no DEFAULT value.", + NameStr(att->attname)))); + } + } fieldno++; /* Done if we hit EOL instead of a delim */ diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index cb0096aeb6..207a29e5d5 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -47,6 +47,8 @@ typedef struct CopyFormatOptions char *null_print; /* NULL marker string (server encoding!) */ int null_print_len; /* length of same */ char *null_print_client; /* same converted to file encoding */ + char *default_print; /* DEFAULT marker string */ + int default_print_len; /* length of same */ char *delim; /* column delimiter (must be 1 byte) */ char *quote; /* CSV quote char (must be 1 byte) */ char *escape; /* CSV escape char (must be 1 byte) */ @@ -79,7 +81,7 @@ extern void EndCopyFrom(CopyFromState cstate); extern bool NextCopyFrom(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls); extern bool NextCopyFromRawFields(CopyFromState cstate, - char ***fields, int *nfields); + char ***fields, int *nfields, bool *defaults); extern void CopyFromErrorCallback(void *arg); extern uint64 CopyFrom(CopyFromState cstate); diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index e37c6032ae..61a658bead 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -87,11 +87,11 @@ typedef struct CopyFromStateData */ MemoryContext copycontext; /* per-copy execution context */ - AttrNumber num_defaults; + AttrNumber num_defaults; /* count of att that are missing and have default value */ FmgrInfo *in_functions; /* array of input functions for each attrs */ Oid *typioparams; /* array of element types for in_functions */ - int *defmap; /* array of default att numbers */ - ExprState **defexprs; /* array of default att expressions */ + int *defmap; /* array of default att numbers related to missing att */ + ExprState **defexprs; /* array of default att expressions for all att */ bool volatile_defexprs; /* is any of defexprs volatile? */ List *range_table; ExprState *qualexpr; diff --git a/src/test/regress/expected/copydefault.out b/src/test/regress/expected/copydefault.out new file mode 100644 index 0000000000..63f727472b --- /dev/null +++ b/src/test/regress/expected/copydefault.out @@ -0,0 +1,76 @@ +-- +-- COPY DEFAULT +-- this file is responsible for testing DEFAULT option of COPY FROM +-- +create temp table copy_default ( + id integer primary key, + text_value text not null default 'test', + ts_value timestamp without time zone not null default '2022-07-05' +); +-- if DEFAULT is not specified, then it will behave as a regular COPY FROM +-- to maintain backward compatibility +copy copy_default from stdin; +select id, text_value, ts_value from copy_default; + id | text_value | ts_value +----+------------+-------------------------- + 1 | value | Mon Jul 04 00:00:00 2022 + 2 | D | Tue Jul 05 00:00:00 2022 +(2 rows) + +truncate copy_default; +copy copy_default from stdin with (format csv); +select id, text_value, ts_value from copy_default; + id | text_value | ts_value +----+------------+-------------------------- + 1 | value | Mon Jul 04 00:00:00 2022 + 2 | \D | Tue Jul 05 00:00:00 2022 +(2 rows) + +truncate copy_default; +-- DEFAULT cannot be used in binary mode +copy copy_default from stdin with (format binary, default '\D'); +ERROR: cannot specify DEFAULT in BINARY mode +-- DEFAULT cannot be new line nor carriage return +copy copy_default from stdin with (default E'\n'); +ERROR: COPY default representation cannot use newline or carriage return +copy copy_default from stdin with (default E'\r'); +ERROR: COPY default representation cannot use newline or carriage return +-- DELIMITER cannot appear in DEFAULT spec +copy copy_default from stdin with (delimiter ';', default 'test;test'); +ERROR: COPY delimiter must not appear in the DEFAULT specification +-- CSV quote cannot appear in DEFAULT spec +copy copy_default from stdin with (format csv, quote '"', default 'test"test'); +ERROR: CSV quote character must not appear in the DEFAULT specification +-- NULL and DEFAULT spec must be different +copy copy_default from stdin with (default '\N'); +ERROR: NULL specification and DEFAULT specification cannot be the same +-- cannot use DEFAULT marker in column that has no DEFAULT value +copy copy_default from stdin with (default '\D'); +ERROR: unexpected DEFAULT in COPY data +DETAIL: Column "id" has no DEFAULT value. +CONTEXT: COPY copy_default, line 1: "\D value '2022-07-04'" +copy copy_default from stdin with (format csv, default '\D'); +ERROR: unexpected DEFAULT in COPY data +DETAIL: Column "id" has no DEFAULT value. +CONTEXT: COPY copy_default, line 1: "\D,value,2022-07-04" +-- successful usage of DEFAULT option in COPY +copy copy_default from stdin with (default '\D'); +select id, text_value, ts_value from copy_default; + id | text_value | ts_value +----+------------+-------------------------- + 1 | value | Mon Jul 04 00:00:00 2022 + 2 | test | Sun Jul 03 00:00:00 2022 + 3 | test | Tue Jul 05 00:00:00 2022 +(3 rows) + +truncate copy_default; +copy copy_default from stdin with (format csv, default '\D'); +select id, text_value, ts_value from copy_default; + id | text_value | ts_value +----+------------+-------------------------- + 1 | value | Mon Jul 04 00:00:00 2022 + 2 | test | Sun Jul 03 00:00:00 2022 + 3 | test | Tue Jul 05 00:00:00 2022 +(3 rows) + +truncate copy_default; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 103e11483d..bfc10d10a0 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -41,7 +41,7 @@ test: geometry horology tstypes regex type_sanity opr_sanity misc_sanity comment # execute two copy tests in parallel, to check that copy itself # is concurrent safe. # ---------- -test: copy copyselect copydml insert insert_conflict +test: copy copyselect copydml insert insert_conflict copydefault # ---------- # More groups of parallel tests diff --git a/src/test/regress/sql/copydefault.sql b/src/test/regress/sql/copydefault.sql new file mode 100644 index 0000000000..1be2c729ae --- /dev/null +++ b/src/test/regress/sql/copydefault.sql @@ -0,0 +1,78 @@ +-- +-- COPY DEFAULT +-- this file is responsible for testing DEFAULT option of COPY FROM +-- + +create temp table copy_default ( + id integer primary key, + text_value text not null default 'test', + ts_value timestamp without time zone not null default '2022-07-05' +); + +-- if DEFAULT is not specified, then it will behave as a regular COPY FROM +-- to maintain backward compatibility +copy copy_default from stdin; +1 value '2022-07-04' +2 \D '2022-07-05' +\. + +select id, text_value, ts_value from copy_default; + +truncate copy_default; + +copy copy_default from stdin with (format csv); +1,value,2022-07-04 +2,\D,2022-07-05 +\. + +select id, text_value, ts_value from copy_default; + +truncate copy_default; + +-- DEFAULT cannot be used in binary mode +copy copy_default from stdin with (format binary, default '\D'); + +-- DEFAULT cannot be new line nor carriage return +copy copy_default from stdin with (default E'\n'); +copy copy_default from stdin with (default E'\r'); + +-- DELIMITER cannot appear in DEFAULT spec +copy copy_default from stdin with (delimiter ';', default 'test;test'); + +-- CSV quote cannot appear in DEFAULT spec +copy copy_default from stdin with (format csv, quote '"', default 'test"test'); + +-- NULL and DEFAULT spec must be different +copy copy_default from stdin with (default '\N'); + +-- cannot use DEFAULT marker in column that has no DEFAULT value +copy copy_default from stdin with (default '\D'); +\D value '2022-07-04' +2 \D '2022-07-05' +\. + +copy copy_default from stdin with (format csv, default '\D'); +\D,value,2022-07-04 +2,\D,2022-07-05 +\. + +-- successful usage of DEFAULT option in COPY +copy copy_default from stdin with (default '\D'); +1 value '2022-07-04' +2 \D '2022-07-03' +3 \D \D +\. + +select id, text_value, ts_value from copy_default; + +truncate copy_default; + +copy copy_default from stdin with (format csv, default '\D'); +1,value,2022-07-04 +2,\D,2022-07-03 +3,\D,\D +\. + +select id, text_value, ts_value from copy_default; + +truncate copy_default; -- 2.31.1