From a40adad6e24d8b4cdfc8ec26749a5bf32915716a Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy Date: Wed, 28 Feb 2024 06:24:54 +0000 Subject: [PATCH v3] Add detailed info when COPY skips soft errors This commit emits individual info like line number and column name when COPY skips soft errors. Because, the summary containing the total rows skipped isn't enough for the users to know what exactly are the malformed rows in the input data. It emits individual info and summary at NOTICE and INFO level respectively to let users switch of individual info by changing client_min_messages to WARNING. Also, one can get all of these information into server logs by changing log_min_messages. --- doc/src/sgml/ref/copy.sgml | 5 ++++- src/backend/commands/copyfrom.c | 2 +- src/backend/commands/copyfromparse.c | 8 ++++++++ src/test/regress/expected/copy2.out | 6 +++++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 55764fc1f2..c633ad5aa3 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -397,7 +397,10 @@ COPY { table_name [ ( FORMAT is text or csv. - A NOTICE message containing the ignored row count is emitted at the end + When ignore option is specified, a + NOTICE message containing the line number and column + name is emitted for each discarded row, and INFO + message containing the ignored row count is emitted at the end of the COPY FROM if at least one row was discarded. diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 1fe70b9133..e11c2d1cff 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1314,7 +1314,7 @@ CopyFrom(CopyFromState cstate) if (cstate->opts.on_error != COPY_ON_ERROR_STOP && cstate->num_errors > 0) - ereport(NOTICE, + ereport(INFO, errmsg_plural("%llu row was skipped due to data type incompatibility", "%llu rows were skipped due to data type incompatibility", (unsigned long long) cstate->num_errors, diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 7cacd0b752..12e604acfa 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -968,7 +968,15 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, (Node *) cstate->escontext, &values[m])) { + Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); cstate->num_errors++; + + ereport(NOTICE, + errmsg("detected data type incompatibility at line number %llu for column %s; COPY %s", + (unsigned long long) cstate->cur_lineno, + cstate->cur_attname, + cstate->cur_relname)); + return true; } diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 25c401ce34..15a1da2eac 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -730,7 +730,11 @@ COPY check_ign_err FROM STDIN WITH (on_error stop); ERROR: invalid input syntax for type integer: "a" CONTEXT: COPY check_ign_err, line 2, column n: "a" COPY check_ign_err FROM STDIN WITH (on_error ignore); -NOTICE: 4 rows were skipped due to data type incompatibility +NOTICE: detected data type incompatibility at line number 2 for column n; COPY check_ign_err +NOTICE: detected data type incompatibility at line number 3 for column k; COPY check_ign_err +NOTICE: detected data type incompatibility at line number 4 for column m; COPY check_ign_err +NOTICE: detected data type incompatibility at line number 5 for column n; COPY check_ign_err +INFO: 4 rows were skipped due to data type incompatibility SELECT * FROM check_ign_err; n | m | k ---+-----+--- -- 2.34.1