From ad5a855ceac2a2bf762009210d1f8193dda58fb3 Mon Sep 17 00:00:00 2001 From: jian he Date: Tue, 19 Nov 2024 12:22:18 +0800 Subject: [PATCH v13 3/3] Add option force_array for COPY JSON FORMAT force_array option can only be used in COPY TO with JSON format. it make the output json output behave like json array type. discussion: https://postgr.es/m/CALvfUkBxTYy5uWPFVwpk_7ii2zgT07t3d-yR_cy4sfrrLU%3Dkcg%40mail.gmail.com discussion: https://postgr.es/m/6a04628d-0d53-41d9-9e35-5a8dc302c34c@joeconway.com --- doc/src/sgml/ref/copy.sgml | 14 ++++++++++++++ src/backend/commands/copy.c | 13 +++++++++++++ src/backend/commands/copyto.c | 28 ++++++++++++++++++++++++++++ src/bin/psql/tab-complete.in.c | 2 +- src/include/commands/copy.h | 1 + src/test/regress/expected/copy.out | 23 +++++++++++++++++++++++ src/test/regress/sql/copy.sql | 9 +++++++++ 7 files changed, 89 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 5bf0f38d90..50cebec0ce 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -43,6 +43,7 @@ COPY { table_name [ ( column_name [, ...] ) | * } FORCE_NOT_NULL { ( column_name [, ...] ) | * } FORCE_NULL { ( column_name [, ...] ) | * } + FORCE_ARRAY [ boolean ] ON_ERROR error_action REJECT_LIMIT maxerror ENCODING 'encoding_name' @@ -392,6 +393,19 @@ COPY { table_name [ ( + + FORCE_ARRAY + + + Force output of square brackets as array decorations at the beginning + and end of output, and commas between the rows. It is allowed only in + COPY TO, and only when using + JSON format. The default is + false. + + + + ON_ERROR diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 4b8bc87666..71091e1bf3 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -490,6 +490,7 @@ ProcessCopyOptions(ParseState *pstate, bool on_error_specified = false; bool log_verbosity_specified = false; bool reject_limit_specified = false; + bool force_array_specified = false; ListCell *option; /* Support external use for option sanity checking */ @@ -644,6 +645,13 @@ ProcessCopyOptions(ParseState *pstate, defel->defname), parser_errposition(pstate, defel->location))); } + else if (strcmp(defel->defname, "force_array") == 0) + { + if (force_array_specified) + errorConflictingDefElem(defel, pstate); + force_array_specified = true; + opts_out->force_array = defGetBoolean(defel); + } else if (strcmp(defel->defname, "on_error") == 0) { if (on_error_specified) @@ -893,6 +901,11 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY json mode cannot be used with %s", "COPY FROM"))); + if (opts_out->format != COPY_FORMAT_JSON && opts_out->force_array) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY %s can only used with JSON mode", "FORCE_ARRAY"))); + if (opts_out->default_print) { if (!is_from) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 87709d76be..7d22ea7e8a 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -81,6 +81,7 @@ typedef struct CopyToStateData List *attnumlist; /* integer list of attnums to copy */ char *filename; /* filename, or NULL for STDOUT */ bool is_program; /* is 'filename' a program to popen? */ + bool json_row_delim_needed; /* need delimiter to start next json array element */ copy_data_dest_cb data_dest_cb; /* function for writing data */ CopyFormatOptions opts; @@ -858,6 +859,15 @@ DoCopyTo(CopyToState cstate) CopySendEndOfRow(cstate); } + /* + * If JSON has been requested, and FORCE_ARRAY has been specified send + * the opening bracket. + */ + if (cstate->opts.format == COPY_FORMAT_JSON && cstate->opts.force_array) + { + CopySendChar(cstate, '['); + CopySendEndOfRow(cstate); + } } if (cstate->rel) @@ -905,6 +915,15 @@ DoCopyTo(CopyToState cstate) CopySendEndOfRow(cstate); } + /* + * If JSON has been requested, and FORCE_ARRAY has been specified send the + * closing bracket. + */ + if (cstate->opts.format == COPY_FORMAT_JSON && cstate->opts.force_array) + { + CopySendChar(cstate, ']'); + CopySendEndOfRow(cstate); + } MemoryContextDelete(cstate->rowcontext); if (fe_copy) @@ -1008,6 +1027,15 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) result = makeStringInfo(); composite_to_json(rowdata, result, false); + if (cstate->json_row_delim_needed && cstate->opts.force_array) + CopySendChar(cstate, ','); + else if (cstate->opts.force_array) + { + /* first row needs no delimiter */ + CopySendChar(cstate, ' '); + cstate->json_row_delim_needed = true; + } + CopySendData(cstate, result->data, result->len); } diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index 48cf854a1d..f291e7caba 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -3234,7 +3234,7 @@ match_previous_words(int pattern_id, else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(")) COMPLETE_WITH("FORMAT", "FREEZE", "DELIMITER", "NULL", "HEADER", "QUOTE", "ESCAPE", "FORCE_QUOTE", - "FORCE_NOT_NULL", "FORCE_NULL", "ENCODING", "DEFAULT", + "FORCE_NOT_NULL", "FORCE_NULL", "FORCE_ARRAY", "ENCODING", "DEFAULT", "ON_ERROR", "LOG_VERBOSITY"); /* Complete COPY FROM|TO filename WITH (FORMAT */ diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 076ae59f96..25e534b901 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -92,6 +92,7 @@ typedef struct CopyFormatOptions List *force_null; /* list of column names */ bool force_null_all; /* FORCE_NULL *? */ bool *force_null_flags; /* per-column CSV FN flags */ + bool force_array; /* add JSON array decorations */ bool convert_selectively; /* do selective binary conversion? */ CopyOnErrorChoice on_error; /* what to do when error happened */ CopyLogVerbosityChoice log_verbosity; /* verbosity of logged messages */ diff --git a/src/test/regress/expected/copy.out b/src/test/regress/expected/copy.out index 430f11f3f1..a35ffbe683 100644 --- a/src/test/regress/expected/copy.out +++ b/src/test/regress/expected/copy.out @@ -96,6 +96,29 @@ ERROR: cannot specify DEFAULT in JSON mode copy copytest from stdin(format json); ERROR: COPY json mode cannot be used with COPY FROM -- all of the above should yield error +--Error +copy copytest to stdout (format csv, force_array true); +ERROR: COPY FORCE_ARRAY can only used with JSON mode +--ok +copy copytest to stdout (format json, force_array); +[ + {"style":"DOS","test":"abc\r\ndef","filler":1} +,{"style":"Unix","test":"abc\ndef","filler":2} +,{"style":"Mac","test":"abc\rdef","filler":3} +,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} +] +copy copytest to stdout (format json, force_array true); +[ + {"style":"DOS","test":"abc\r\ndef","filler":1} +,{"style":"Unix","test":"abc\ndef","filler":2} +,{"style":"Mac","test":"abc\rdef","filler":3} +,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} +] +copy copytest to stdout (format json, force_array false); +{"style":"DOS","test":"abc\r\ndef","filler":1} +{"style":"Unix","test":"abc\ndef","filler":2} +{"style":"Mac","test":"abc\rdef","filler":3} +{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4} -- embedded escaped characters create temp table copyjsontest ( id bigserial, diff --git a/src/test/regress/sql/copy.sql b/src/test/regress/sql/copy.sql index 3d21f20c98..91daf8482c 100644 --- a/src/test/regress/sql/copy.sql +++ b/src/test/regress/sql/copy.sql @@ -95,6 +95,15 @@ copy copytest to stdout (format json, default '|'); copy copytest from stdin(format json); -- all of the above should yield error +--Error +copy copytest to stdout (format csv, force_array true); + +--ok +copy copytest to stdout (format json, force_array); + +copy copytest to stdout (format json, force_array true); + +copy copytest to stdout (format json, force_array false); -- embedded escaped characters create temp table copyjsontest ( id bigserial, -- 2.34.1