agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v10 4/9] Add pg_ls_dir_metadata to list a dir with file metadata.. 30+ messages / 3 participants [nested] [flat]
* [PATCH v10 4/9] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-08 22:15 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-08 22:15 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- src/backend/utils/adt/genfile.c | 184 +++++++++++++++++--------------- src/include/catalog/pg_proc.dat | 6 ++ 2 files changed, 106 insertions(+), 84 deletions(-) diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index bcf9bd1b97..4f437573ab 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,13 +36,15 @@ #include "utils/syscache.h" #include "utils/timestamp.h" -typedef struct -{ - char *location; - DIR *dirdesc; - bool include_dot_dirs; -} directory_fctx; +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); +#define FLAG_ISDIR (1<<0) /* Show column: isdir */ +#define FLAG_METADATA (1<<1) /* Show columns: mtime, size */ +#define FLAG_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define FLAG_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define FLAG_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define FLAG_SKIP_DIRS (1<<5) /* Do not show directories */ +#define FLAG_SKIP_SPECIAL (1<<6) /* Do not show special file types */ /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -447,67 +449,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; - struct dirent *de; - directory_fctx *fctx; - MemoryContext oldcontext; - - if (SRF_IS_FIRSTCALL()) - { - bool missing_ok = false; - bool include_dot_dirs = false; - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - funcctx = SRF_FIRSTCALL_INIT(); - oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - - fctx = palloc(sizeof(directory_fctx)); - fctx->location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - fctx->include_dot_dirs = include_dot_dirs; - fctx->dirdesc = AllocateDir(fctx->location); - - if (!fctx->dirdesc) - { - if (missing_ok && errno == ENOENT) - { - MemoryContextSwitchTo(oldcontext); - SRF_RETURN_DONE(funcctx); - } - else - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open directory \"%s\": %m", - fctx->location))); - } - funcctx->user_fctx = fctx; - MemoryContextSwitchTo(oldcontext); - } - - funcctx = SRF_PERCALL_SETUP(); - fctx = (directory_fctx *) funcctx->user_fctx; - - while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL) - { - if (!fctx->include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - SRF_RETURN_NEXT(funcctx, CStringGetTextDatum(de->d_name)); - } - - FreeDir(fctx->dirdesc); - - SRF_RETURN_DONE(funcctx); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, FLAG_SKIP_DOT_DIRS); } /* @@ -520,7 +464,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, FLAG_SKIP_DOT_DIRS); } /* @@ -530,7 +476,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -539,6 +485,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&FLAG_ISDIR) || (flags&FLAG_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&FLAG_ISDIR) || !(flags&FLAG_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= FLAG_MISSING_OK; + else + flags &= ~FLAG_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~FLAG_SKIP_DOT_DIRS; + else + flags |= FLAG_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -554,8 +526,18 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags&FLAG_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } else { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -574,7 +556,7 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags&FLAG_MISSING_OK && errno == ENOENT) { tuplestore_donestoring(tupstore); return (Datum) 0; @@ -584,13 +566,19 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; + if (flags&FLAG_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + /* Skip hidden files */ - if (de->d_name[0] == '.') + if (flags&FLAG_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -600,13 +588,26 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", path))); - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + if (S_ISDIR(attrib.st_mode)) + { + if (flags&FLAG_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags&FLAG_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & FLAG_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & FLAG_ISDIR) + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -621,14 +622,16 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, + FLAG_SKIP_DIRS|FLAG_SKIP_HIDDEN|FLAG_SKIP_SPECIAL|FLAG_METADATA); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, + FLAG_SKIP_DIRS|FLAG_SKIP_HIDDEN|FLAG_SKIP_SPECIAL|FLAG_METADATA); } /* @@ -646,7 +649,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + FLAG_MISSING_OK|FLAG_SKIP_DIRS|FLAG_SKIP_HIDDEN|FLAG_SKIP_SPECIAL|FLAG_METADATA); } /* @@ -675,5 +679,17 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + FLAG_MISSING_OK|FLAG_SKIP_DIRS|FLAG_SKIP_HIDDEN|FLAG_SKIP_SPECIAL|FLAG_METADATA); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, FLAG_METADATA|FLAG_SKIP_SPECIAL|FLAG_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7fb574f9dc..0a1859f709 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10741,6 +10741,12 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool bool', + proallargtypes => '{text,bool,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,dir_ok,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', -- 2.17.0 --rCwQ2Y43eQY6RBgR Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0005-pg_ls_tmpdir-to-show-isdir-argument.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v9 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-08 22:15 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-08 22:15 UTC (permalink / raw) Need catversion bumped? --- src/backend/utils/adt/genfile.c | 51 ++++++++++++++++++++++++++------- src/include/catalog/pg_proc.dat | 6 ++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 897b11a77d..4699aea568 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -524,7 +524,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) /* Generic function to return a directory listing of files */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok, bool dir_ok) { FuncCallContext *funcctx; struct dirent *de; @@ -540,13 +540,17 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) fctx = palloc(sizeof(directory_fctx)); - tupdesc = CreateTemplateTupleDesc(3); + tupdesc = CreateTemplateTupleDesc(dir_ok ? 4:3); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "name", TEXTOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "size", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 3, "modification", TIMESTAMPTZOID, -1, 0); + if (dir_ok) + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "isdir", + BOOLOID, -1, 0); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); fctx->location = pstrdup(dir); @@ -575,8 +579,8 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; HeapTuple tuple; @@ -592,13 +596,21 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", path))); - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) + if (S_ISDIR(attrib.st_mode)) + { + if (!dir_ok) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + /* Ignore anything but regular files */ continue; values[0] = CStringGetTextDatum(de->d_name); values[1] = Int64GetDatum((int64) attrib.st_size); values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (dir_ok) + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + memset(nulls, 0, sizeof(nulls)); tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); @@ -613,14 +625,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, false, false); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, false, false); } /* @@ -638,7 +650,7 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, true, false); } /* @@ -667,5 +679,24 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true, false); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + bool missing_ok = false; + bool include_dot_dirs = false; + + if (!PG_ARGISNULL(1)) + missing_ok = PG_GETARG_BOOL(1); + if (!PG_ARGISNULL(2)) + /* XXX: Not implemented */ + include_dot_dirs = PG_GETARG_BOOL(2); + + return pg_ls_dir_files(fcinfo, dirname, missing_ok, true); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7fb574f9dc..0a1859f709 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10741,6 +10741,12 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool bool', + proallargtypes => '{text,bool,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,dir_ok,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', -- 2.17.0 --32u276st3Jlj2kUU Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0005-Add-tests-exercizing-pg_ls_tmpdir.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v11 4/9] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-08 22:15 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-08 22:15 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 17 +++ src/backend/utils/adt/genfile.c | 190 ++++++++++++++++++-------------- src/include/catalog/pg_proc.dat | 6 + 3 files changed, 129 insertions(+), 84 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index fc4d7f0f78..dfea9f8061 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21342,6 +21342,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); List the contents of a directory. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_metadata(<parameter>dirname</parameter> <type>text</type> [, <parameter>missing_ok</parameter> <type>boolean</type>, <parameter>include_dot_dirs</parameter> <type>boolean</type>])</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21442,6 +21451,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <para> + <function>pg_ls_dir_metadata</function> lists the files in the specified + directory along with the file's metadata. + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index bcf9bd1b97..b105363903 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,14 +36,23 @@ #include "utils/syscache.h" #include "utils/timestamp.h" -typedef struct -{ - char *location; - DIR *dirdesc; - bool include_dot_dirs; -} directory_fctx; +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) + /* * Convert a "text" filename argument to C string, and check it's allowable. * @@ -447,67 +456,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; - struct dirent *de; - directory_fctx *fctx; - MemoryContext oldcontext; - - if (SRF_IS_FIRSTCALL()) - { - bool missing_ok = false; - bool include_dot_dirs = false; - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - funcctx = SRF_FIRSTCALL_INIT(); - oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - - fctx = palloc(sizeof(directory_fctx)); - fctx->location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - fctx->include_dot_dirs = include_dot_dirs; - fctx->dirdesc = AllocateDir(fctx->location); - - if (!fctx->dirdesc) - { - if (missing_ok && errno == ENOENT) - { - MemoryContextSwitchTo(oldcontext); - SRF_RETURN_DONE(funcctx); - } - else - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open directory \"%s\": %m", - fctx->location))); - } - funcctx->user_fctx = fctx; - MemoryContextSwitchTo(oldcontext); - } - - funcctx = SRF_PERCALL_SETUP(); - fctx = (directory_fctx *) funcctx->user_fctx; - - while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL) - { - if (!fctx->include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - SRF_RETURN_NEXT(funcctx, CStringGetTextDatum(de->d_name)); - } - - FreeDir(fctx->dirdesc); - - SRF_RETURN_DONE(funcctx); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -520,7 +471,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -530,7 +483,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -539,6 +492,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -554,8 +533,18 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } else { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -574,7 +563,7 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags&LS_DIR_MISSING_OK && errno == ENOENT) { tuplestore_donestoring(tupstore); return (Datum) 0; @@ -584,13 +573,19 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + /* Skip hidden files */ - if (de->d_name[0] == '.') + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -600,13 +595,26 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", path))); - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -621,14 +629,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -646,7 +654,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC|LS_DIR_MISSING_OK); } /* @@ -675,5 +684,18 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC|LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA|LS_DIR_SKIP_SPECIAL|LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7fb574f9dc..0a1859f709 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10741,6 +10741,12 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool bool', + proallargtypes => '{text,bool,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,dir_ok,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', -- 2.17.0 --oTHb8nViIGeoXxdp Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v11-0005-pg_ls_tmpdir-to-show-isdir-argument.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v12 05/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-08 22:15 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-08 22:15 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 17 +++ src/backend/utils/adt/genfile.c | 194 ++++++++++++++++++-------------- src/include/catalog/pg_proc.dat | 6 + 3 files changed, 132 insertions(+), 85 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 2c6142a0e0..4b966ed847 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21342,6 +21342,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); List the contents of a directory. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_metadata(<parameter>dirname</parameter> <type>text</type> [, <parameter>missing_ok</parameter> <type>boolean</type>, <parameter>include_dot_dirs</parameter> <type>boolean</type>])</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21442,6 +21451,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <para> + <function>pg_ls_dir_metadata</function> lists the files in the specified + directory along with the file's metadata. + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index bcf9bd1b97..98ab9a2b92 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,14 +36,23 @@ #include "utils/syscache.h" #include "utils/timestamp.h" -typedef struct -{ - char *location; - DIR *dirdesc; - bool include_dot_dirs; -} directory_fctx; +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) + /* * Convert a "text" filename argument to C string, and check it's allowable. * @@ -447,67 +456,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; - struct dirent *de; - directory_fctx *fctx; - MemoryContext oldcontext; - - if (SRF_IS_FIRSTCALL()) - { - bool missing_ok = false; - bool include_dot_dirs = false; - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - funcctx = SRF_FIRSTCALL_INIT(); - oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - - fctx = palloc(sizeof(directory_fctx)); - fctx->location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - fctx->include_dot_dirs = include_dot_dirs; - fctx->dirdesc = AllocateDir(fctx->location); - - if (!fctx->dirdesc) - { - if (missing_ok && errno == ENOENT) - { - MemoryContextSwitchTo(oldcontext); - SRF_RETURN_DONE(funcctx); - } - else - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open directory \"%s\": %m", - fctx->location))); - } - funcctx->user_fctx = fctx; - MemoryContextSwitchTo(oldcontext); - } - - funcctx = SRF_PERCALL_SETUP(); - fctx = (directory_fctx *) funcctx->user_fctx; - - while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL) - { - if (!fctx->include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - SRF_RETURN_NEXT(funcctx, CStringGetTextDatum(de->d_name)); - } - - FreeDir(fctx->dirdesc); - - SRF_RETURN_DONE(funcctx); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -520,7 +471,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -530,7 +483,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -539,6 +492,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -554,8 +533,18 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } else { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -574,7 +563,7 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags&LS_DIR_MISSING_OK && errno == ENOENT) { tuplestore_donestoring(tupstore); return (Datum) 0; @@ -584,13 +573,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -600,13 +596,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", path))); - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -621,14 +631,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -646,7 +656,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC|LS_DIR_MISSING_OK); } /* @@ -675,5 +686,18 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC|LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA|LS_DIR_SKIP_SPECIAL|LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7fb574f9dc..0a1859f709 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10741,6 +10741,12 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool bool', + proallargtypes => '{text,bool,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,dir_ok,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', -- 2.17.0 --wIc/V6YLA2QdyfT4 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0006-Show-links-to-dirs-with-isdir-false.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v13 3/8] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-08 22:15 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-08 22:15 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Change to use lstat() to allow pg_ls_dir_recurse to avoid infinite recursion. That means: - links to dirs are shown with isdir=false; - timestamps shown are those of the link; - changed pg_stat_file for consistency; Need catversion bumped? --- doc/src/sgml/func.sgml | 19 ++- src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 218 +++++++++++++++------------ src/include/catalog/pg_proc.dat | 6 + 4 files changed, 150 insertions(+), 94 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 2c6142a0e0..68c7327e1d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21342,6 +21342,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); List the contents of a directory. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_metadata(<parameter>dirname</parameter> <type>text</type> [, <parameter>missing_ok</parameter> <type>boolean</type>, <parameter>include_dot_dirs</parameter> <type>boolean</type>])</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21442,6 +21451,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <para> + <function>pg_ls_dir_metadata</function> lists the files in the specified + directory along with the file's metadata. + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> @@ -21528,7 +21545,7 @@ SELECT convert_from(pg_read_binary_file('file_in_utf8.txt'), 'UTF8'); size, last accessed time stamp, last modified time stamp, last file status change time stamp (Unix platforms only), file creation time stamp (Windows only), and a <type>boolean</type> - indicating if it is a directory (or a symbolic link to a directory). + indicating if it is a directory (and not a symbolic link to a directory). Typical usages include: <programlisting> SELECT * FROM pg_stat_file('filename'); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b8a3f46912..05a644a7c9 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1435,6 +1435,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 01185f218b..4b70a00a35 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -370,7 +385,7 @@ pg_stat_file(PG_FUNCTION_ARGS) filename = convert_and_check_filename(filename_t); - if (stat(filename, &fst) < 0) + if (lstat(filename, &fst) < 0) { if (missing_ok && errno == ENOENT) PG_RETURN_NULL(); @@ -413,6 +428,10 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + if (pgwin32_is_junction(path)) /* Links are not directories */ + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +459,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +474,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +486,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +495,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +535,18 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } else { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,36 +565,67 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags&LS_DIR_MISSING_OK && errno == ENOENT) + { + tuplestore_donestoring(tupstore); return (Datum) 0; + } /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ snprintf(path, sizeof(path), "%s/%s", dir, de->d_name); - if (stat(path, &attrib) < 0) + if (lstat(path, &attrib) < 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", path))); - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links are not directories */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -621,14 +639,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -646,7 +664,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC|LS_DIR_MISSING_OK); } /* @@ -675,5 +694,18 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC|LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA|LS_DIR_SKIP_SPECIAL|LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7fb574f9dc..0a1859f709 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10741,6 +10741,12 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool bool', + proallargtypes => '{text,bool,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,dir_ok,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', -- 2.17.0 --E0h0CbphJD8hN+Gf Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0004-pg_ls_tmpdir-to-show-isdir-argument.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v14 3/8] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Change to use lstat() to allow pg_ls_dir_recurse to avoid infinite recursion. That means: - links to dirs are shown with isdir=false; - timestamps shown are those of the link; - changed pg_stat_file for consistency; Need catversion bumped? --- doc/src/sgml/func.sgml | 19 +- src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 244 ++++++++++++------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 230 insertions(+), 94 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 2c6142a0e0..68c7327e1d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21342,6 +21342,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); List the contents of a directory. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_metadata(<parameter>dirname</parameter> <type>text</type> [, <parameter>missing_ok</parameter> <type>boolean</type>, <parameter>include_dot_dirs</parameter> <type>boolean</type>])</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21442,6 +21451,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <para> + <function>pg_ls_dir_metadata</function> lists the files in the specified + directory along with the file's metadata. + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> @@ -21528,7 +21545,7 @@ SELECT convert_from(pg_read_binary_file('file_in_utf8.txt'), 'UTF8'); size, last accessed time stamp, last modified time stamp, last file status change time stamp (Unix platforms only), file creation time stamp (Windows only), and a <type>boolean</type> - indicating if it is a directory (or a symbolic link to a directory). + indicating if it is a directory (and not a symbolic link to a directory). Typical usages include: <programlisting> SELECT * FROM pg_stat_file('filename'); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b8a3f46912..05a644a7c9 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1435,6 +1435,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 01185f218b..daae06aa9a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -370,7 +385,7 @@ pg_stat_file(PG_FUNCTION_ARGS) filename = convert_and_check_filename(filename_t); - if (stat(filename, &fst) < 0) + if (lstat(filename, &fst) < 0) { if (missing_ok && errno == ENOENT) PG_RETURN_NULL(); @@ -413,6 +428,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +460,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +475,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +487,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +496,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +536,18 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } else { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,36 +566,77 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags&LS_DIR_MISSING_OK && errno == ENOENT) + { + tuplestore_donestoring(tupstore); return (Datum) 0; + } /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; + struct stat lattrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ snprintf(path, sizeof(path), "%s/%s", dir, de->d_name); - if (stat(path, &attrib) < 0) + if (stat(path, &attrib) < 0 || + lstat(path, &lattrib) < 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", path))); - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(lattrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (S_ISDIR(attrib.st_mode)) + { + /* + * Do nothing: links to dirs are not skipped (but are shown as + * isdir=false) + */ + ; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) lattrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(lattrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(lattrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -621,14 +650,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -646,7 +675,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -675,5 +705,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7fb574f9dc..b1a957fe3e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10741,6 +10741,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '5033', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Kynn+LdAwU9N+JqL Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v14-0004-pg_ls_tmpdir-to-show-isdir-argument.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v26 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 233 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 222 insertions(+), 93 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 2707e757ca..79d30a1568 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25799,6 +25799,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b140c210bc..93d4f1e4a6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1506,6 +1506,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..198a99391a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,17 +511,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 22970f46cd..17b03d1abd 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11266,6 +11266,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index edbfd9abc1..fd7c3c791f 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -225,6 +225,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index b32a3a4b74..64d9d70874 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -68,6 +68,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --yKkOmjQZXRsvHRX8 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v26-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v27 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 233 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 222 insertions(+), 93 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 977b75a531..76ae2f2dd8 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25793,6 +25793,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 5f2541d316..fcd050b752 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1607,6 +1607,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 322152ebd9..f59f853983 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +683,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +708,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,5 +738,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4309fa40dd..b85bee2d1e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11517,6 +11517,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index ea0fc48dbd..b4112ec298 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -238,6 +238,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index c133e73499..a32212be04 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 1bbe7e0323..5f7ce3d09f 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index eb6ac12ab4..c169e527d9 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -77,6 +77,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --19uQFt6ulqmgNgg1 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v27-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d9b3598977..fc1b4ac98c 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25714,6 +25714,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 2bd5f5ea14..1c77430f0c 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1467,6 +1467,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 219ac160f8..4824a55480 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -413,6 +428,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +460,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +475,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +487,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +496,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +536,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,20 +568,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -606,13 +603,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -626,14 +644,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -651,7 +669,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -680,5 +699,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4bce3ad8de..9f46cba5ed 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10904,6 +10904,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '5033', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v15 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir XXX: Change to use lstat() to allow pg_ls_dir_recurse to avoid infinite recursion. That means: - links to dirs are NOT SHOWN (was: shown with isdir=false;) - NO: timestamps shown are those of the link; - changed pg_stat_file for consistency; Need catversion bumped? --- doc/src/sgml/func.sgml | 17 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 227 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 214 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 630f03d7dd..ed8e1977ba 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21372,6 +21372,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); List the contents of a directory. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_metadata(<parameter>dirname</parameter> <type>text</type> [, <parameter>missing_ok</parameter> <type>boolean</type>, <parameter>include_dot_dirs</parameter> <type>boolean</type>])</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21472,6 +21481,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <para> + <function>pg_ls_dir_metadata</function> lists the files in the specified + directory along with the file's metadata. + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 83d00c6cde..4511f9d658 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1436,6 +1436,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 219ac160f8..eaa7c548e0 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -413,6 +428,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +460,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +475,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +487,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +496,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +536,18 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } else { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,20 +566,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags&LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -606,13 +601,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -626,14 +642,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -651,7 +667,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -680,5 +697,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index a6a708cca9..e8846e1522 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10801,6 +10801,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '5033', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --8w3uRX/HFJGApMzv Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v15-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v24 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 231 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 221 insertions(+), 92 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 507bc1a668..5e267c4f8d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25757,6 +25757,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 2e4aa1c4b6..e7647787cf 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1503,6 +1503,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..7351615f8a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,17 +511,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* * Generic function to return a directory listing of files. * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index e7fbda9f81..f35a684dcf 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10975,6 +10975,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index edbfd9abc1..fd7c3c791f 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -225,6 +225,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index b32a3a4b74..64d9d70874 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -68,6 +68,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --mPTHnM80CEnHQ2WJ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v24-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v19 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index b7c450ea29..2931f9d7d5 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25711,6 +25711,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 5314e9348f..f5217a0d93 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1481,6 +1481,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 219ac160f8..4824a55480 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -413,6 +428,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +460,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +475,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +487,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +496,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +536,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,20 +568,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -606,13 +603,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -626,14 +644,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -651,7 +669,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -680,5 +699,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 61f2c2f5b4..5b0147ee2d 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10902,6 +10902,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --S0GG+JvAI2G0KxBG Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v19-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v21 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 959f6a1c2f..eead88419f 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25741,6 +25741,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 8625cbeab6..b9b2a6aa20 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1478,6 +1478,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..24d858d1bc 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,7 +511,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -571,7 +523,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 95604e988a..83927b510a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10919,6 +10919,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..20bfe71822 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..68e2bc6586 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Tcb1KvpfnM4LxW2s Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v21-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v18 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7c06afd3ea..8ea064144f 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25714,6 +25714,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 56420bbc9d..cc8e870740 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 219ac160f8..4824a55480 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -413,6 +428,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +460,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +475,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +487,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +496,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +536,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,20 +568,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -606,13 +603,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -626,14 +644,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -651,7 +669,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -680,5 +699,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 61f2c2f5b4..5b0147ee2d 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10902,6 +10902,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --5I6of5zJg18YgZEa Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v18-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v23 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7b1dc264f6..0a924fdcc3 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 2e4aa1c4b6..e7647787cf 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1503,6 +1503,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..6f003bbf67 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,7 +511,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -571,7 +523,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index c01da4bf01..add3c7dc62 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10971,6 +10971,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..20bfe71822 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..68e2bc6586 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --mhjHhnbe5PrRcwjY Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v23-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v36 3/7] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 205 ++++++++++++------- src/include/catalog/pg_proc.dat | 12 ++ src/test/regress/expected/misc_functions.out | 24 +++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 217 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 933118203f4..ba30faa8441 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25940,6 +25940,27 @@ SELECT collation for ('foo' COLLATE "de_DE"); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 73da687d5dd..3fa6f9af0a8 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -700,6 +700,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 2bf52192567..c7322ed405a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything beginning with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -446,6 +461,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -473,54 +493,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - DIR *dirdesc; - struct dirent *de; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, - values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -533,23 +508,55 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; DIR *dirdesc; struct dirent *de; - SetSingleFuncCall(fcinfo, 0); + /* isdir depends on metadata */ + Assert(!(flags & LS_DIR_ISDIR) || (flags & LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags & LS_DIR_ISDIR) || !(flags & LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + /* missing_ok */ + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + /* include_dot_dirs */ + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } + + if (flags & LS_DIR_METADATA) + SetSingleFuncCall(fcinfo, 0); + else + SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); /* * Now walk the directory. Note that we must do this within a single SRF @@ -560,20 +567,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -588,13 +602,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -608,14 +644,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -633,7 +669,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -662,7 +699,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -671,7 +736,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -680,7 +745,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -705,5 +770,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 87aa571a331..0e5c977e69f 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11815,6 +11815,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 45544469af5..e54e38f54ad 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -475,6 +475,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index c52cf1cfcf9..8159c9f18f1 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -33,6 +33,14 @@ SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN' pg_tblspc/NNN (1 row) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 59d6e517503..fcb456f434d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -155,6 +155,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 21db433f2a8..cf683c3bf3a 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -27,6 +27,11 @@ CREATE TABLESPACE regress_tblspace LOCATION ''; SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN') FROM pg_tablespace WHERE spcname = 'regress_tblspace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.1 --4ybNbZnZ8tziJ7D6 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v36-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v37 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 205 ++++++++++++------- src/include/catalog/pg_proc.dat | 12 ++ src/test/regress/expected/misc_functions.out | 24 +++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 217 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d958c3e74ac..a80d794a43a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25900,6 +25900,27 @@ LOG: Grand total: 1651920 bytes in 201 blocks; 622360 free (88 chunks); 1029560 </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 30a048f6b09..b77b83017cb 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -704,6 +704,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index ab6f67f8747..c041c7630c8 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything beginning with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -516,6 +531,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -543,54 +563,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - DIR *dirdesc; - struct dirent *de; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - InitMaterializedSRF(fcinfo, MAT_SRF_USE_EXPECTED_DESC); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, - values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -603,23 +578,55 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; DIR *dirdesc; struct dirent *de; - InitMaterializedSRF(fcinfo, 0); + /* isdir depends on metadata */ + Assert(!(flags & LS_DIR_ISDIR) || (flags & LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags & LS_DIR_ISDIR) || !(flags & LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + /* missing_ok */ + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + /* include_dot_dirs */ + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } + + if (flags & LS_DIR_METADATA) + InitMaterializedSRF(fcinfo, 0); + else + InitMaterializedSRF(fcinfo, MAT_SRF_USE_EXPECTED_DESC); /* * Now walk the directory. Note that we must do this within a single SRF @@ -630,20 +637,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -658,13 +672,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -678,14 +714,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -703,7 +739,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -732,7 +769,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -741,7 +806,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -750,7 +815,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -775,5 +840,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 20f5aa56eab..89892647808 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11758,6 +11758,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 77d285ecc85..a656f9ad07e 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -539,6 +539,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index c52cf1cfcf9..8159c9f18f1 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -33,6 +33,14 @@ SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN' pg_tblspc/NNN (1 row) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index d299f3d8949..f808555382d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -181,6 +181,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 21db433f2a8..cf683c3bf3a 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -27,6 +27,11 @@ CREATE TABLESPACE regress_tblspace LOCATION ''; SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN') FROM pg_tablespace WHERE spcname = 'regress_tblspace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.25.1 --Pk/CTwBz1VvfPIDp Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v37-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v33 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 240 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 226 insertions(+), 96 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d32b414e04f..14b8fba3123 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25958,6 +25958,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index fd1421788e6..2ffc2f9d1a6 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -702,6 +702,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 542bbacaa24..3921723fda6 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +684,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +709,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,7 +739,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -728,7 +776,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -737,7 +785,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -762,5 +810,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0859dc81cac..b58987b713c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11661,6 +11661,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index b7264af3ecb..a627839bc28 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -309,6 +309,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index 2dfbcfdebe1..c6b9074ab02 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -24,6 +24,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION ''; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 169f66416c3..bfa8e309e9a 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -109,6 +109,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 896f05cea32..3aadac8b611 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -23,6 +23,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION ''; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.1 --9CzcV6dAFIr7O1Ie Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v33-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v34 03/15] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 203 ++++++++++++------- src/include/catalog/pg_proc.dat | 12 ++ src/test/regress/expected/misc_functions.out | 24 +++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 215 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 0be4743e3ed..c2ff924d021 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25976,6 +25976,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 81bac6f5812..b4d3609cce7 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -700,6 +700,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 1ed01620a1b..1878bac7eb3 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything beginning with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -446,6 +461,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -473,54 +493,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - DIR *dirdesc; - struct dirent *de; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, - values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -533,23 +508,53 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; DIR *dirdesc; struct dirent *de; - SetSingleFuncCall(fcinfo, 0); + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } + + if (flags & LS_DIR_METADATA) + SetSingleFuncCall(fcinfo, 0); + else + SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); /* * Now walk the directory. Note that we must do this within a single SRF @@ -560,20 +565,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -588,13 +600,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -608,14 +642,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -633,7 +667,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -662,7 +697,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -671,7 +734,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -680,7 +743,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -705,5 +768,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d8e8715ed1c..43ee0cdc454 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11677,6 +11677,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index b08e7c4a6d0..d18f401248a 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -452,6 +452,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index 2dfbcfdebe1..c6b9074ab02 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -24,6 +24,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION ''; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index ebb0352ac68..05794e1ae45 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -147,6 +147,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 896f05cea32..3aadac8b611 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -23,6 +23,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION ''; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.1 --smOfPzt+Qjm5bNGJ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v34-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v35 3/7] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 205 ++++++++++++------- src/include/catalog/pg_proc.dat | 12 ++ src/test/regress/expected/misc_functions.out | 24 +++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 217 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d01aeec9f88..d2a455a3e27 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25987,6 +25987,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 81bac6f5812..b4d3609cce7 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -700,6 +700,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 88f279d1b31..75b7bf99849 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything beginning with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -446,6 +461,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -473,54 +493,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - DIR *dirdesc; - struct dirent *de; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, - values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -533,23 +508,55 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; DIR *dirdesc; struct dirent *de; - SetSingleFuncCall(fcinfo, 0); + /* isdir depends on metadata */ + Assert(!(flags & LS_DIR_ISDIR) || (flags & LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags & LS_DIR_ISDIR) || !(flags & LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + /* missing_ok */ + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + /* include_dot_dirs */ + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } + + if (flags & LS_DIR_METADATA) + SetSingleFuncCall(fcinfo, 0); + else + SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); /* * Now walk the directory. Note that we must do this within a single SRF @@ -560,20 +567,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -588,13 +602,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -608,14 +644,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -633,7 +669,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -662,7 +699,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -671,7 +736,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -680,7 +745,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -705,5 +770,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 25304430f44..7307f44f371 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11769,6 +11769,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 45544469af5..e54e38f54ad 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -475,6 +475,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index c52cf1cfcf9..8159c9f18f1 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -33,6 +33,14 @@ SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN' pg_tblspc/NNN (1 row) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 59d6e517503..fcb456f434d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -155,6 +155,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 21db433f2a8..cf683c3bf3a 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -27,6 +27,11 @@ CREATE TABLESPACE regress_tblspace LOCATION ''; SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN') FROM pg_tablespace WHERE spcname = 'regress_tblspace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.1 --olLTNZSltDMg5Vbm Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v35-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v30 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 233 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 222 insertions(+), 93 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7a830f0684..dd9b1afde4 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25848,6 +25848,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index a416e94d37..649a78605a 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -698,6 +698,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index c436d9318b..8efb798b2f 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -451,6 +466,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -478,79 +498,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -563,17 +513,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -582,6 +534,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -596,8 +574,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -616,20 +606,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -644,13 +641,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -664,14 +682,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -689,7 +707,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -718,5 +737,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fde251fa4f..b405aa341c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11547,6 +11547,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index ea0fc48dbd..b4112ec298 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -238,6 +238,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index c133e73499..a32212be04 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 1bbe7e0323..5f7ce3d09f 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index eb6ac12ab4..c169e527d9 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -77,6 +77,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --ZwgA9U+XZDXt4+m+ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v30-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v31 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 239 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 225 insertions(+), 96 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d36479d86d..e0099e77df 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25860,6 +25860,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index f6789025a5..a45bac89ff 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -698,6 +698,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 027ed86400..0728547ec2 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +683,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +708,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,7 +738,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -728,7 +775,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -737,7 +784,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -762,5 +809,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index e934361dc3..39fbaacbc9 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11645,6 +11645,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 830de507e7..b3a9d11b5c 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -267,6 +267,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index cb9774ecc8..e69fa17004 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index e7629d470e..c2f6d64c5c 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 422a1369ae..ca946d08bd 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -99,6 +99,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- -- 2.17.0 --qZVVwWJgpX9Jzs7f Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v31-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v32 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 239 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 225 insertions(+), 96 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index b5c1befe627..7948e6ec06c 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25945,6 +25945,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 3a4fa9091b1..5eb6cc8572c 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -702,6 +702,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 027ed864001..0728547ec2b 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +683,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +708,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,7 +738,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -728,7 +775,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -737,7 +784,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -762,5 +809,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4d992dc2241..c99e1633bbf 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11645,6 +11645,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 830de507e7c..b3a9d11b5c0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -267,6 +267,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index 864f4b6e208..21f7f19b67f 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -16,6 +16,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION :'testtablespace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 422a1369aeb..ca946d08bde 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -99,6 +99,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 92076db9a13..291ca029c14 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -16,6 +16,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION :'testtablespace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.0 --Bne5rrxQd65beI7a Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v32-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v28 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 233 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 222 insertions(+), 93 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index aa0dcde886..507a6d73f8 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25821,6 +25821,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 451db2ee0a..dd84b0f11a 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1632,6 +1632,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 322152ebd9..f59f853983 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +683,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +708,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,5 +738,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index f4957653ae..550fbf734a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11534,6 +11534,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index ea0fc48dbd..b4112ec298 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -238,6 +238,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index c133e73499..a32212be04 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 1bbe7e0323..5f7ce3d09f 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index eb6ac12ab4..c169e527d9 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -77,6 +77,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --vk/v8fjDPiDepTtA Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v28-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v20 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 959f6a1c2f..eead88419f 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25741,6 +25741,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b6d35c2d11..360b6bda26 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1478,6 +1478,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 46d0977c2e..3324ffe16a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,7 +511,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -571,7 +523,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 95604e988a..186ced4a35 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10919,6 +10919,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Z1Z8UV8BNhgCynIS Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v20-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v22 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7ef2ec9972..34c92903d8 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25747,6 +25747,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index c6dd084fbc..169602c1b6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1509,6 +1509,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..054b8d4b1a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,7 +511,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -571,7 +523,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 24ec2cfed6..02b1436d83 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10964,6 +10964,18 @@ proallargtypes => '{regclass,text,text,int8}', proargmodes => '{i,i,o,o}', proargnames => '{relation,fork,path,failed_block_num}', prosrc => 'pg_relation_check_pages' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..20bfe71822 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..68e2bc6586 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --d6Gm4EdcadzBjdND Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v22-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v25 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 231 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 221 insertions(+), 92 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index df29af6371..226923a134 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25750,6 +25750,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b140c210bc..93d4f1e4a6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1506,6 +1506,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..7351615f8a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,17 +511,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* * Generic function to return a directory listing of files. * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fc2202b843..8e88c549d9 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10976,6 +10976,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index edbfd9abc1..fd7c3c791f 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -225,6 +225,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index b32a3a4b74..64d9d70874 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -68,6 +68,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --SBikYMzjhZGK9d4p Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v25-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v16 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 19 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 218 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 96b08d0500..9d617f95b9 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25300,6 +25300,17 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); List the contents of a directory. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_metadata(<parameter>dirname</parameter> <type>text</type> [, <parameter>missing_ok</parameter> <type>boolean</type>, <parameter>include_dot_dirs</parameter> <type>boolean</type>])</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + For each file in a directory, list the file and its metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25400,6 +25411,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <para> + <function>pg_ls_dir_metadata</function> lists the files in the specified + directory along with the file's metadata. + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 2bd5f5ea14..1c77430f0c 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1467,6 +1467,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 219ac160f8..4824a55480 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -413,6 +428,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +460,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +475,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +487,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +496,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +536,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,20 +568,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -606,13 +603,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -626,14 +644,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -651,7 +669,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -680,5 +699,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4bce3ad8de..9f46cba5ed 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10904,6 +10904,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '5033', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v48 3/7] Make archiver process an auxiliary process @ 2020-03-13 07:59 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Kyotaro Horiguchi @ 2020-03-13 07:59 UTC (permalink / raw) This is a preliminary patch for shared-memory based stats collector. Archiver process must be a auxiliary process since it uses shared memory after stats data was moved into shared-memory. Make the process an auxiliary process in order to make it work. --- src/backend/access/transam/xlogarchive.c | 6 +- src/backend/bootstrap/bootstrap.c | 22 ++-- src/backend/postmaster/pgarch.c | 130 +++-------------------- src/backend/postmaster/postmaster.c | 50 +++++---- src/backend/storage/lmgr/proc.c | 1 + src/include/access/xlog.h | 3 + src/include/access/xlogarchive.h | 1 + src/include/miscadmin.h | 2 + src/include/postmaster/pgarch.h | 4 +- src/include/storage/pmsignal.h | 1 - src/include/storage/proc.h | 3 + 11 files changed, 69 insertions(+), 154 deletions(-) diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index 1c5a4f8b5a..d01859bde5 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -29,7 +29,9 @@ #include "storage/fd.h" #include "storage/ipc.h" #include "storage/lwlock.h" +#include "storage/latch.h" #include "storage/pmsignal.h" +#include "storage/proc.h" /* * Attempt to retrieve the specified file from off-line archival storage. @@ -490,8 +492,8 @@ XLogArchiveNotify(const char *xlog) } /* Notify archiver that it's got something to do */ - if (IsUnderPostmaster) - SendPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER); + if (IsUnderPostmaster && ProcGlobal->archiverLatch) + SetLatch(ProcGlobal->archiverLatch); } /* diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 6f615e6622..41da0c5059 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -317,6 +317,9 @@ AuxiliaryProcessMain(int argc, char *argv[]) case StartupProcess: MyBackendType = B_STARTUP; break; + case ArchiverProcess: + MyBackendType = B_ARCHIVER; + break; case BgWriterProcess: MyBackendType = B_BG_WRITER; break; @@ -437,30 +440,29 @@ AuxiliaryProcessMain(int argc, char *argv[]) proc_exit(1); /* should never return */ case StartupProcess: - /* don't set signals, startup process has its own agenda */ StartupProcessMain(); - proc_exit(1); /* should never return */ + proc_exit(1); + + case ArchiverProcess: + PgArchiverMain(); + proc_exit(1); case BgWriterProcess: - /* don't set signals, bgwriter has its own agenda */ BackgroundWriterMain(); - proc_exit(1); /* should never return */ + proc_exit(1); case CheckpointerProcess: - /* don't set signals, checkpointer has its own agenda */ CheckpointerMain(); - proc_exit(1); /* should never return */ + proc_exit(1); case WalWriterProcess: - /* don't set signals, walwriter has its own agenda */ InitXLOGAccess(); WalWriterMain(); - proc_exit(1); /* should never return */ + proc_exit(1); case WalReceiverProcess: - /* don't set signals, walreceiver has its own agenda */ WalReceiverMain(); - proc_exit(1); /* should never return */ + proc_exit(1); default: elog(PANIC, "unrecognized process type: %d", (int) MyAuxProcType); diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index edec311f12..9a2e21bf86 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -48,6 +48,7 @@ #include "storage/latch.h" #include "storage/pg_shmem.h" #include "storage/pmsignal.h" +#include "storage/procsignal.h" #include "utils/guc.h" #include "utils/ps_status.h" @@ -78,13 +79,11 @@ * Local data * ---------- */ -static time_t last_pgarch_start_time; static time_t last_sigterm_time = 0; /* * Flags set by interrupt handlers for later service in the main loop. */ -static volatile sig_atomic_t wakened = false; static volatile sig_atomic_t ready_to_stop = false; /* ---------- @@ -95,8 +94,6 @@ static volatile sig_atomic_t ready_to_stop = false; static pid_t pgarch_forkexec(void); #endif -NON_EXEC_STATIC void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn(); -static void pgarch_waken(SIGNAL_ARGS); static void pgarch_waken_stop(SIGNAL_ARGS); static void pgarch_MainLoop(void); static void pgarch_ArchiverCopyLoop(void); @@ -110,75 +107,6 @@ static void pgarch_archiveDone(char *xlog); * ------------------------------------------------------------ */ -/* - * pgarch_start - * - * Called from postmaster at startup or after an existing archiver - * died. Attempt to fire up a fresh archiver process. - * - * Returns PID of child process, or 0 if fail. - * - * Note: if fail, we will be called again from the postmaster main loop. - */ -int -pgarch_start(void) -{ - time_t curtime; - pid_t pgArchPid; - - /* - * Do nothing if no archiver needed - */ - if (!XLogArchivingActive()) - return 0; - - /* - * Do nothing if too soon since last archiver start. This is a safety - * valve to protect against continuous respawn attempts if the archiver is - * dying immediately at launch. Note that since we will be re-called from - * the postmaster main loop, we will get another chance later. - */ - curtime = time(NULL); - if ((unsigned int) (curtime - last_pgarch_start_time) < - (unsigned int) PGARCH_RESTART_INTERVAL) - return 0; - last_pgarch_start_time = curtime; - -#ifdef EXEC_BACKEND - switch ((pgArchPid = pgarch_forkexec())) -#else - switch ((pgArchPid = fork_process())) -#endif - { - case -1: - ereport(LOG, - (errmsg("could not fork archiver: %m"))); - return 0; - -#ifndef EXEC_BACKEND - case 0: - /* in postmaster child ... */ - InitPostmasterChild(); - - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - - /* Drop our connection to postmaster's shared memory, as well */ - dsm_detach_all(); - PGSharedMemoryDetach(); - - PgArchiverMain(0, NULL); - break; -#endif - - default: - return (int) pgArchPid; - } - - /* shouldn't get here */ - return 0; -} - /* ------------------------------------------------------------ * Local functions called by archiver follow * ------------------------------------------------------------ @@ -212,14 +140,9 @@ pgarch_forkexec(void) #endif /* EXEC_BACKEND */ -/* - * PgArchiverMain - * - * The argc/argv parameters are valid only in EXEC_BACKEND case. However, - * since we don't use 'em, it hardly matters... - */ -NON_EXEC_STATIC void -PgArchiverMain(int argc, char *argv[]) +/* Main entry point for archiver process */ +void +PgArchiverMain(void) { /* * Ignore all signals usually bound to some action in the postmaster, @@ -231,33 +154,26 @@ PgArchiverMain(int argc, char *argv[]) /* SIGQUIT handler was already set up by InitPostmasterChild */ pqsignal(SIGALRM, SIG_IGN); pqsignal(SIGPIPE, SIG_IGN); - pqsignal(SIGUSR1, pgarch_waken); + pqsignal(SIGUSR1, procsignal_sigusr1_handler); pqsignal(SIGUSR2, pgarch_waken_stop); + /* Reset some signals that are accepted by postmaster but not here */ pqsignal(SIGCHLD, SIG_DFL); + + /* Unblock signals (they were blocked when the postmaster forked us) */ PG_SETMASK(&UnBlockSig); - MyBackendType = B_ARCHIVER; - init_ps_display(NULL); + /* + * Advertise our latch that backends can use to wake us up while we're + * sleeping. + */ + ProcGlobal->archiverLatch = &MyProc->procLatch; pgarch_MainLoop(); exit(0); } -/* SIGUSR1 signal handler for archiver process */ -static void -pgarch_waken(SIGNAL_ARGS) -{ - int save_errno = errno; - - /* set flag that there is work to be done */ - wakened = true; - SetLatch(MyLatch); - - errno = save_errno; -} - /* SIGUSR2 signal handler for archiver process */ static void pgarch_waken_stop(SIGNAL_ARGS) @@ -282,14 +198,6 @@ pgarch_MainLoop(void) pg_time_t last_copy_time = 0; bool time_to_stop; - /* - * We run the copy loop immediately upon entry, in case there are - * unarchived files left over from a previous database run (or maybe the - * archiver died unexpectedly). After that we wait for a signal or - * timeout before doing more. - */ - wakened = true; - /* * There shouldn't be anything for the archiver to do except to wait for a * signal ... however, the archiver exists to protect our data, so she @@ -328,12 +236,8 @@ pgarch_MainLoop(void) } /* Do what we're here for */ - if (wakened || time_to_stop) - { - wakened = false; - pgarch_ArchiverCopyLoop(); - last_copy_time = time(NULL); - } + pgarch_ArchiverCopyLoop(); + last_copy_time = time(NULL); /* * Sleep until a signal is received, or until a poll is forced by @@ -354,13 +258,9 @@ pgarch_MainLoop(void) WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, timeout * 1000L, WAIT_EVENT_ARCHIVER_MAIN); - if (rc & WL_TIMEOUT) - wakened = true; if (rc & WL_POSTMASTER_DEATH) time_to_stop = true; } - else - wakened = true; } /* diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index edab95a19e..22e1cd244c 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -548,6 +548,7 @@ static void ShmemBackendArrayRemove(Backend *bn); #endif /* EXEC_BACKEND */ #define StartupDataBase() StartChildProcess(StartupProcess) +#define StartArchiver() StartChildProcess(ArchiverProcess) #define StartBackgroundWriter() StartChildProcess(BgWriterProcess) #define StartCheckpointer() StartChildProcess(CheckpointerProcess) #define StartWalWriter() StartChildProcess(WalWriterProcess) @@ -1792,7 +1793,7 @@ ServerLoop(void) /* If we have lost the archiver, try to start a new one. */ if (PgArchPID == 0 && PgArchStartupAllowed()) - PgArchPID = pgarch_start(); + PgArchPID = StartArchiver(); /* If we need to signal the autovacuum launcher, do so now */ if (avlauncher_needs_signal) @@ -3007,7 +3008,7 @@ reaper(SIGNAL_ARGS) if (!IsBinaryUpgrade && AutoVacuumingActive() && AutoVacPID == 0) AutoVacPID = StartAutoVacLauncher(); if (PgArchStartupAllowed() && PgArchPID == 0) - PgArchPID = pgarch_start(); + PgArchPID = StartArchiver(); if (PgStatPID == 0) PgStatPID = pgstat_start(); @@ -3142,20 +3143,16 @@ reaper(SIGNAL_ARGS) } /* - * Was it the archiver? If so, just try to start a new one; no need - * to force reset of the rest of the system. (If fail, we'll try - * again in future cycles of the main loop.). Unless we were waiting - * for it to shut down; don't restart it in that case, and - * PostmasterStateMachine() will advance to the next shutdown step. + * Was it the archiver? Normal exit can be ignored; we'll start a new + * one at the next iteration of the postmaster's main loop, if + * necessary. Any other exit condition is treated as a crash. */ if (pid == PgArchPID) { PgArchPID = 0; if (!EXIT_STATUS_0(exitstatus)) - LogChildExit(LOG, _("archiver process"), - pid, exitstatus); - if (PgArchStartupAllowed()) - PgArchPID = pgarch_start(); + HandleChildCrash(pid, exitstatus, + _("archiver process")); continue; } @@ -3403,7 +3400,7 @@ CleanupBackend(int pid, /* * HandleChildCrash -- cleanup after failed backend, bgwriter, checkpointer, - * walwriter, autovacuum, or background worker. + * walwriter, autovacuum, archiver or background worker. * * The objectives here are to clean up our local state about the child * process, and to signal all other remaining children to quickdie. @@ -3609,6 +3606,18 @@ HandleChildCrash(int pid, int exitstatus, const char *procname) signal_child(AutoVacPID, (SendStop ? SIGSTOP : SIGQUIT)); } + /* Take care of the archiver too */ + if (pid == PgArchPID) + PgArchPID = 0; + else if (PgArchPID != 0 && take_action) + { + ereport(DEBUG2, + (errmsg_internal("sending %s to process %d", + (SendStop ? "SIGSTOP" : "SIGQUIT"), + (int) PgArchPID))); + signal_child(PgArchPID, (SendStop ? SIGSTOP : SIGQUIT)); + } + /* * Force a power-cycle of the pgarch process too. (This isn't absolutely * necessary, but it seems like a good idea for robustness, and it @@ -3912,6 +3921,7 @@ PostmasterStateMachine(void) Assert(CheckpointerPID == 0); Assert(WalWriterPID == 0); Assert(AutoVacPID == 0); + Assert(PgArchPID == 0); /* syslogger is not considered here */ pmState = PM_NO_CHILDREN; } @@ -5140,7 +5150,7 @@ sigusr1_handler(SIGNAL_ARGS) */ Assert(PgArchPID == 0); if (XLogArchivingAlways()) - PgArchPID = pgarch_start(); + PgArchPID = StartArchiver(); /* * If we aren't planning to enter hot standby mode later, treat @@ -5194,16 +5204,6 @@ sigusr1_handler(SIGNAL_ARGS) if (StartWorkerNeeded || HaveCrashedWorker) maybe_start_bgworkers(); - if (CheckPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER) && - PgArchPID != 0) - { - /* - * Send SIGUSR1 to archiver process, to wake it up and begin archiving - * next WAL file. - */ - signal_child(PgArchPID, SIGUSR1); - } - /* Tell syslogger to rotate logfile if requested */ if (SysLoggerPID != 0) { @@ -5445,6 +5445,10 @@ StartChildProcess(AuxProcType type) ereport(LOG, (errmsg("could not fork startup process: %m"))); break; + case ArchiverProcess: + ereport(LOG, + (errmsg("could not fork archiver process: %m"))); + break; case BgWriterProcess: ereport(LOG, (errmsg("could not fork background writer process: %m"))); diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 897045ee27..bf5d6da89b 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -182,6 +182,7 @@ InitProcGlobal(void) ProcGlobal->startupBufferPinWaitBufId = -1; ProcGlobal->walwriterLatch = NULL; ProcGlobal->checkpointerLatch = NULL; + ProcGlobal->archiverLatch = NULL; pg_atomic_init_u32(&ProcGlobal->procArrayGroupFirst, INVALID_PGPROCNO); pg_atomic_init_u32(&ProcGlobal->clogGroupFirst, INVALID_PGPROCNO); diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 75ec1073bd..551f518cc2 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -339,6 +339,9 @@ extern XLogRecPtr GetRedoRecPtr(void); extern XLogRecPtr GetInsertRecPtr(void); extern XLogRecPtr GetFlushRecPtr(void); extern XLogRecPtr GetLastImportantRecPtr(void); +extern void XLogArchiveWakeupStart(void); +extern void XLogArchiveWakeupEnd(void); +extern void XLogArchiveWakeup(void); extern void RemovePromoteSignalFiles(void); extern bool PromoteIsTriggered(void); diff --git a/src/include/access/xlogarchive.h b/src/include/access/xlogarchive.h index 3edd1a976c..1a59181cf9 100644 --- a/src/include/access/xlogarchive.h +++ b/src/include/access/xlogarchive.h @@ -25,6 +25,7 @@ extern void ExecuteRecoveryCommand(const char *command, const char *commandName, extern void KeepFileRestoredFromArchive(const char *path, const char *xlogfname); extern void XLogArchiveNotify(const char *xlog); extern void XLogArchiveNotifySeg(XLogSegNo segno); +extern void XLogArchiveWakeup(void); extern void XLogArchiveForceDone(const char *xlog); extern bool XLogArchiveCheckDone(const char *xlog); extern bool XLogArchiveIsBusy(const char *xlog); diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 1bdc97e308..adb9f819bb 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -419,6 +419,7 @@ typedef enum BootstrapProcess, StartupProcess, BgWriterProcess, + ArchiverProcess, CheckpointerProcess, WalWriterProcess, WalReceiverProcess, @@ -431,6 +432,7 @@ extern AuxProcType MyAuxProcType; #define AmBootstrapProcess() (MyAuxProcType == BootstrapProcess) #define AmStartupProcess() (MyAuxProcType == StartupProcess) #define AmBackgroundWriterProcess() (MyAuxProcType == BgWriterProcess) +#define AmArchiverProcess() (MyAuxProcType == ArchiverProcess) #define AmCheckpointerProcess() (MyAuxProcType == CheckpointerProcess) #define AmWalWriterProcess() (MyAuxProcType == WalWriterProcess) #define AmWalReceiverProcess() (MyAuxProcType == WalReceiverProcess) diff --git a/src/include/postmaster/pgarch.h b/src/include/postmaster/pgarch.h index d102a21ab7..385b002dfe 100644 --- a/src/include/postmaster/pgarch.h +++ b/src/include/postmaster/pgarch.h @@ -32,8 +32,6 @@ */ extern int pgarch_start(void); -#ifdef EXEC_BACKEND -extern void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn(); -#endif +extern void PgArchiverMain(void) pg_attribute_noreturn(); #endif /* _PGARCH_H */ diff --git a/src/include/storage/pmsignal.h b/src/include/storage/pmsignal.h index dbbed18f61..8ed4d87ae6 100644 --- a/src/include/storage/pmsignal.h +++ b/src/include/storage/pmsignal.h @@ -34,7 +34,6 @@ typedef enum { PMSIGNAL_RECOVERY_STARTED, /* recovery has started */ PMSIGNAL_BEGIN_HOT_STANDBY, /* begin Hot Standby */ - PMSIGNAL_WAKEN_ARCHIVER, /* send a NOTIFY signal to xlog archiver */ PMSIGNAL_ROTATE_LOGFILE, /* send SIGUSR1 to syslogger to rotate logfile */ PMSIGNAL_START_AUTOVAC_LAUNCHER, /* start an autovacuum launcher */ PMSIGNAL_START_AUTOVAC_WORKER, /* start an autovacuum worker */ diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index a777cb64a1..491747c113 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -357,6 +357,9 @@ typedef struct PROC_HDR int startupProcPid; /* Buffer id of the buffer that Startup process waits for pin on, or -1 */ int startupBufferPinWaitBufId; + /* Archiver process's latch */ + Latch *archiverLatch; + /* Current shared estimate of appropriate spins_per_delay value */ } PROC_HDR; extern PGDLLIMPORT PROC_HDR *ProcGlobal; -- 2.27.0 ----Next_Part(Fri_Mar__5_17_18_56_2021_497)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v48-0004-Shared-memory-based-stats-collector.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v27 5/9] Row pattern recognition patch (executor). @ 2024-12-30 23:53 Tatsuo Ishii <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Tatsuo Ishii @ 2024-12-30 23:53 UTC (permalink / raw) --- src/backend/executor/nodeWindowAgg.c | 1845 +++++++++++++++++++++++++- src/backend/utils/adt/windowfuncs.c | 37 +- src/include/catalog/pg_proc.dat | 6 + src/include/nodes/execnodes.h | 58 + 4 files changed, 1935 insertions(+), 11 deletions(-) diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c index 70a7025818..2a06676572 100644 --- a/src/backend/executor/nodeWindowAgg.c +++ b/src/backend/executor/nodeWindowAgg.c @@ -36,6 +36,7 @@ #include "access/htup_details.h" #include "catalog/objectaccess.h" #include "catalog/pg_aggregate.h" +#include "catalog/pg_collation_d.h" #include "catalog/pg_proc.h" #include "executor/executor.h" #include "executor/nodeWindowAgg.h" @@ -45,9 +46,11 @@ #include "optimizer/optimizer.h" #include "parser/parse_agg.h" #include "parser/parse_coerce.h" +#include "regex/regex.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/datum.h" +#include "utils/fmgroids.h" #include "utils/expandeddatum.h" #include "utils/lsyscache.h" #include "utils/memutils.h" @@ -159,6 +162,33 @@ typedef struct WindowStatePerAggData bool restart; /* need to restart this agg in this cycle? */ } WindowStatePerAggData; +/* + * Set of StringInfo. Used in RPR. + */ +#define STRSET_FROZEN (1 << 0) /* string is frozen */ +#define STRSET_DISCARDED (1 << 1) /* string is scheduled to be discarded */ +#define STRSET_MATCHED (1 << 2) /* string is confirmed to be matched + * with pattern */ + +typedef struct StringSet +{ + StringInfo *str_set; + Size set_size; /* current array allocation size in number of + * items */ + int set_index; /* current used size */ + int *info; /* an array of information bit per StringInfo. + * see above */ +} StringSet; + +/* + * Structure used by check_rpr_navigation() and rpr_navigation_walker(). + */ +typedef struct NavigationInfo +{ + bool is_prev; /* true if PREV */ + int num_vars; /* number of var nodes */ +} NavigationInfo; + static void initialize_windowaggregate(WindowAggState *winstate, WindowStatePerFunc perfuncstate, WindowStatePerAgg peraggstate); @@ -184,6 +214,7 @@ static void release_partition(WindowAggState *winstate); static int row_is_in_frame(WindowAggState *winstate, int64 pos, TupleTableSlot *slot); + static void update_frameheadpos(WindowAggState *winstate); static void update_frametailpos(WindowAggState *winstate); static void update_grouptailpos(WindowAggState *winstate); @@ -195,9 +226,60 @@ static Datum GetAggInitVal(Datum textInitVal, Oid transtype); static bool are_peers(WindowAggState *winstate, TupleTableSlot *slot1, TupleTableSlot *slot2); + +static int WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot, + int relpos, int seektype, bool set_mark, + bool *isnull, bool *isout); static bool window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot); +static void attno_map(Node *node); +static bool attno_map_walker(Node *node, void *context); +static int row_is_in_reduced_frame(WindowObject winobj, int64 pos); +static bool rpr_is_defined(WindowAggState *winstate); + +static void create_reduced_frame_map(WindowAggState *winstate); +static int get_reduced_frame_map(WindowAggState *winstate, int64 pos); +static void register_reduced_frame_map(WindowAggState *winstate, int64 pos, + int val); +static void clear_reduced_frame_map(WindowAggState *winstate); +static void update_reduced_frame(WindowObject winobj, int64 pos); + +static int64 evaluate_pattern(WindowObject winobj, int64 current_pos, + char *vname, StringInfo encoded_str, bool *result); + +static bool get_slots(WindowObject winobj, int64 current_pos); + +static int search_str_set(WindowAggState *winstate, + StringSet *input_str_set); +static StringSet *generate_patterns(StringSet *input_str_set, + char *pattern, VariablePos *variable_pos, + char tail_pattern_initial); +static int add_pattern(StringInfo old, int old_info, StringSet *new_str_set, + char c, char *pattern, char tail_pattern_initial, + int resultlen); +static int freeze_pattern(StringInfo old, int old_info, + StringSet *new_str_set, + char *pattern, int resultlen); +static char pattern_initial(WindowAggState *winstate, char *vname); +static int do_pattern_match(char *pattern, char *encoded_str, int len); + +static StringSet *string_set_init(void); +static void string_set_add(StringSet *string_set, StringInfo str, int info); +static StringInfo string_set_get(StringSet *string_set, int index, int *flag); +static int string_set_get_size(StringSet *string_set); +static void string_set_discard(StringSet *string_set); +static VariablePos *variable_pos_init(void); +static void variable_pos_register(VariablePos *variable_pos, char initial, + int pos); +static bool variable_pos_compare(VariablePos *variable_pos, + char initial1, char initial2); +static int variable_pos_fetch(VariablePos *variable_pos, char initial, + int index); +static VariablePos *variable_pos_build(WindowAggState *winstate); + +static void check_rpr_navigation(Node *node, bool is_prev); +static bool rpr_navigation_walker(Node *node, void *context); /* * initialize_windowaggregate @@ -774,10 +856,12 @@ eval_windowaggregates(WindowAggState *winstate) * transition function, or * - we have an EXCLUSION clause, or * - if the new frame doesn't overlap the old one + * - if RPR is enabled * * Note that we don't strictly need to restart in the last case, but if * we're going to remove all rows from the aggregation anyway, a restart * surely is faster. + * we restart aggregation too. *---------- */ numaggs_restart = 0; @@ -788,7 +872,8 @@ eval_windowaggregates(WindowAggState *winstate) (winstate->aggregatedbase != winstate->frameheadpos && !OidIsValid(peraggstate->invtransfn_oid)) || (winstate->frameOptions & FRAMEOPTION_EXCLUSION) || - winstate->aggregatedupto <= winstate->frameheadpos) + winstate->aggregatedupto <= winstate->frameheadpos || + rpr_is_defined(winstate)) { peraggstate->restart = true; numaggs_restart++; @@ -862,7 +947,22 @@ eval_windowaggregates(WindowAggState *winstate) * head, so that tuplestore can discard unnecessary rows. */ if (agg_winobj->markptr >= 0) - WinSetMarkPosition(agg_winobj, winstate->frameheadpos); + { + int64 markpos = winstate->frameheadpos; + + if (rpr_is_defined(winstate)) + { + /* + * If RPR is used, it is possible PREV wants to look at the + * previous row. So the mark pos should be frameheadpos - 1 + * unless it is below 0. + */ + markpos -= 1; + if (markpos < 0) + markpos = 0; + } + WinSetMarkPosition(agg_winobj, markpos); + } /* * Now restart the aggregates that require it. @@ -917,6 +1017,14 @@ eval_windowaggregates(WindowAggState *winstate) { winstate->aggregatedupto = winstate->frameheadpos; ExecClearTuple(agg_row_slot); + + /* + * If RPR is defined, we do not use aggregatedupto_nonrestarted. To + * avoid assertion failure below, we reset aggregatedupto_nonrestarted + * to frameheadpos. + */ + if (rpr_is_defined(winstate)) + aggregatedupto_nonrestarted = winstate->frameheadpos; } /* @@ -930,6 +1038,12 @@ eval_windowaggregates(WindowAggState *winstate) { int ret; +#ifdef RPR_DEBUG + elog(DEBUG1, "===== loop in frame starts: aggregatedupto: " INT64_FORMAT " aggregatedbase: " INT64_FORMAT, + winstate->aggregatedupto, + winstate->aggregatedbase); +#endif + /* Fetch next row if we didn't already */ if (TupIsNull(agg_row_slot)) { @@ -945,9 +1059,53 @@ eval_windowaggregates(WindowAggState *winstate) ret = row_is_in_frame(winstate, winstate->aggregatedupto, agg_row_slot); if (ret < 0) break; + if (ret == 0) goto next_tuple; + if (rpr_is_defined(winstate)) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "reduced_frame_map: %d aggregatedupto: " INT64_FORMAT " aggregatedbase: " INT64_FORMAT, + get_reduced_frame_map(winstate, + winstate->aggregatedupto), + winstate->aggregatedupto, + winstate->aggregatedbase); +#endif + + /* + * If the row status at currentpos is already decided and current + * row status is not decided yet, it means we passed the last + * reduced frame. Time to break the loop. + */ + if (get_reduced_frame_map(winstate, winstate->currentpos) + != RF_NOT_DETERMINED && + get_reduced_frame_map(winstate, winstate->aggregatedupto) + == RF_NOT_DETERMINED) + break; + + /* + * Otherwise we need to calculate the reduced frame. + */ + ret = row_is_in_reduced_frame(winstate->agg_winobj, + winstate->aggregatedupto); + if (ret == -1) /* unmatched row */ + break; + + /* + * Check if current row needs to be skipped due to no match. + */ + if (get_reduced_frame_map(winstate, + winstate->aggregatedupto) == RF_SKIPPED && + winstate->aggregatedupto == winstate->aggregatedbase) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "skip current row for aggregation"); +#endif + break; + } + } + /* Set tuple context for evaluation of aggregate arguments */ winstate->tmpcontext->ecxt_outertuple = agg_row_slot; @@ -976,6 +1134,7 @@ next_tuple: ExecClearTuple(agg_row_slot); } + /* The frame's end is not supposed to move backwards, ever */ Assert(aggregatedupto_nonrestarted <= winstate->aggregatedupto); @@ -1199,6 +1358,7 @@ begin_partition(WindowAggState *winstate) winstate->framehead_valid = false; winstate->frametail_valid = false; winstate->grouptail_valid = false; + create_reduced_frame_map(winstate); winstate->spooled_rows = 0; winstate->currentpos = 0; winstate->frameheadpos = 0; @@ -2170,6 +2330,11 @@ ExecWindowAgg(PlanState *pstate) CHECK_FOR_INTERRUPTS(); +#ifdef RPR_DEBUG + elog(DEBUG1, "ExecWindowAgg called. pos: " INT64_FORMAT, + winstate->currentpos); +#endif + if (winstate->status == WINDOWAGG_DONE) return NULL; @@ -2278,6 +2443,17 @@ ExecWindowAgg(PlanState *pstate) /* don't evaluate the window functions when we're in pass-through mode */ if (winstate->status == WINDOWAGG_RUN) { + /* + * If RPR is defined and skip mode is next row, we need to clear + * existing reduced frame info so that we newly calculate the info + * starting from current row. + */ + if (rpr_is_defined(winstate)) + { + if (winstate->rpSkipTo == ST_NEXT_ROW) + clear_reduced_frame_map(winstate); + } + /* * Evaluate true window functions */ @@ -2444,6 +2620,9 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) TupleDesc scanDesc; ListCell *l; + TargetEntry *te; + Expr *expr; + /* check for unsupported flags */ Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK))); @@ -2542,6 +2721,16 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) winstate->temp_slot_2 = ExecInitExtraTupleSlot(estate, scanDesc, &TTSOpsMinimalTuple); + winstate->prev_slot = ExecInitExtraTupleSlot(estate, scanDesc, + &TTSOpsMinimalTuple); + + winstate->next_slot = ExecInitExtraTupleSlot(estate, scanDesc, + &TTSOpsMinimalTuple); + + winstate->null_slot = ExecInitExtraTupleSlot(estate, scanDesc, + &TTSOpsMinimalTuple); + winstate->null_slot = ExecStoreAllNullTuple(winstate->null_slot); + /* * create frame head and tail slots only if needed (must create slots in * exactly the same cases that update_frameheadpos and update_frametailpos @@ -2723,6 +2912,49 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) winstate->inRangeAsc = node->inRangeAsc; winstate->inRangeNullsFirst = node->inRangeNullsFirst; + /* Set up SKIP TO type */ + winstate->rpSkipTo = node->rpSkipTo; + /* Set up row pattern recognition PATTERN clause */ + winstate->patternVariableList = node->patternVariable; + winstate->patternRegexpList = node->patternRegexp; + + /* Set up row pattern recognition DEFINE clause */ + winstate->defineInitial = node->defineInitial; + winstate->defineVariableList = NIL; + winstate->defineClauseList = NIL; + if (node->defineClause != NIL) + { + /* + * Tweak arg var of PREV/NEXT so that it refers to scan/inner slot. + */ + foreach(l, node->defineClause) + { + char *name; + ExprState *exps; + + te = lfirst(l); + name = te->resname; + expr = te->expr; + +#ifdef RPR_DEBUG + elog(DEBUG1, "defineVariable name: %s", name); +#endif + winstate->defineVariableList = + lappend(winstate->defineVariableList, + makeString(pstrdup(name))); + attno_map((Node *) expr); + exps = ExecInitExpr(expr, (PlanState *) winstate); + winstate->defineClauseList = + lappend(winstate->defineClauseList, exps); + } + } + + /* + * Build variable_pos + */ + if (winstate->defineInitial) + winstate->variable_pos = variable_pos_build(winstate); + winstate->all_first = true; winstate->partition_spooled = false; winstate->more_partitions = false; @@ -2731,6 +2963,111 @@ ExecInitWindowAgg(WindowAgg *node, EState *estate, int eflags) return winstate; } +/* + * Rewrite varno of Var nodes that are the argument of PREV/NET so that they + * see scan tuple (PREV) or inner tuple (NEXT). Also we check the arguments + * of PREV/NEXT include at least 1 column reference. This is required by the + * SQL standard. + */ +static void +attno_map(Node *node) +{ + (void) expression_tree_walker(node, attno_map_walker, NULL); +} + +static bool +attno_map_walker(Node *node, void *context) +{ + FuncExpr *func; + int nargs; + bool is_prev; + + if (node == NULL) + return false; + + if (IsA(node, FuncExpr)) + { + func = (FuncExpr *) node; + + if (func->funcid == F_PREV || func->funcid == F_NEXT) + { + /* + * The SQL standard allows to have two more arguments form of + * PREV/NEXT. But currently we allow only 1 argument form. + */ + nargs = list_length(func->args); + if (list_length(func->args) != 1) + elog(ERROR, "PREV/NEXT must have 1 argument but function %d has %d args", + func->funcid, nargs); + + /* + * Check expr of PREV/NEXT aruguments and replace varno. + */ + is_prev = (func->funcid == F_PREV) ? true : false; + check_rpr_navigation(node, is_prev); + } + } + return expression_tree_walker(node, attno_map_walker, NULL); +} + +/* + * Rewrite varno of Var of RPR navigation operations (PREV/NEXT). + * If is_prev is true, we take care PREV, otherwise NEXT. + */ +static void +check_rpr_navigation(Node *node, bool is_prev) +{ + NavigationInfo context; + + context.is_prev = is_prev; + context.num_vars = 0; + (void) expression_tree_walker(node, rpr_navigation_walker, &context); + if (context.num_vars < 1) + ereport(ERROR, + errmsg("row pattern navigation operation's argument must include at least one column reference")); +} + +static bool +rpr_navigation_walker(Node *node, void *context) +{ + NavigationInfo *nav = (NavigationInfo *) context; + + if (node == NULL) + return false; + + switch (nodeTag(node)) + { + case T_Var: + { + Var *var = (Var *) node; + + nav->num_vars++; + + if (nav->is_prev) + { + /* + * Rewrite varno from OUTER_VAR to regular var no so that + * the var references scan tuple. + */ + var->varno = var->varnosyn; + } + else + var->varno = INNER_VAR; + } + break; + case T_Const: + case T_FuncExpr: + case T_OpExpr: + break; + + default: + ereport(ERROR, + errmsg("row pattern navigation operation's argument includes unsupported expression")); + } + return expression_tree_walker(node, rpr_navigation_walker, context); +} + + /* ----------------- * ExecEndWindowAgg * ----------------- @@ -2788,6 +3125,8 @@ ExecReScanWindowAgg(WindowAggState *node) ExecClearTuple(node->agg_row_slot); ExecClearTuple(node->temp_slot_1); ExecClearTuple(node->temp_slot_2); + ExecClearTuple(node->prev_slot); + ExecClearTuple(node->next_slot); if (node->framehead_slot) ExecClearTuple(node->framehead_slot); if (node->frametail_slot) @@ -3148,7 +3487,8 @@ window_gettupleslot(WindowObject winobj, int64 pos, TupleTableSlot *slot) return false; if (pos < winobj->markpos) - elog(ERROR, "cannot fetch row before WindowObject's mark position"); + elog(ERROR, "cannot fetch row: " INT64_FORMAT " before WindowObject's mark position: " INT64_FORMAT, + pos, winobj->markpos); oldcontext = MemoryContextSwitchTo(winstate->ss.ps.ps_ExprContext->ecxt_per_query_memory); @@ -3468,14 +3808,54 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno, WindowAggState *winstate; ExprContext *econtext; TupleTableSlot *slot; - int64 abs_pos; - int64 mark_pos; Assert(WindowObjectIsValid(winobj)); winstate = winobj->winstate; econtext = winstate->ss.ps.ps_ExprContext; slot = winstate->temp_slot_1; + if (WinGetSlotInFrame(winobj, slot, + relpos, seektype, set_mark, + isnull, isout) == 0) + { + econtext->ecxt_outertuple = slot; + return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno), + econtext, isnull); + } + + if (isout) + *isout = true; + *isnull = true; + return (Datum) 0; +} + +/* + * WinGetSlotInFrame + * slot: TupleTableSlot to store the result + * relpos: signed rowcount offset from the seek position + * seektype: WINDOW_SEEK_HEAD or WINDOW_SEEK_TAIL + * set_mark: If the row is found/in frame and set_mark is true, the mark is + * moved to the row as a side-effect. + * isnull: output argument, receives isnull status of result + * isout: output argument, set to indicate whether target row position + * is out of frame (can pass NULL if caller doesn't care about this) + * + * Returns 0 if we successfullt got the slot. false if out of frame. + * (also isout is set) + */ +static int +WinGetSlotInFrame(WindowObject winobj, TupleTableSlot *slot, + int relpos, int seektype, bool set_mark, + bool *isnull, bool *isout) +{ + WindowAggState *winstate; + int64 abs_pos; + int64 mark_pos; + int num_reduced_frame; + + Assert(WindowObjectIsValid(winobj)); + winstate = winobj->winstate; + switch (seektype) { case WINDOW_SEEK_CURRENT: @@ -3542,11 +3922,25 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno, winstate->frameOptions); break; } + num_reduced_frame = row_is_in_reduced_frame(winobj, + winstate->frameheadpos); + if (num_reduced_frame < 0) + goto out_of_frame; + else if (num_reduced_frame > 0) + if (relpos >= num_reduced_frame) + goto out_of_frame; break; case WINDOW_SEEK_TAIL: /* rejecting relpos > 0 is easy and simplifies code below */ if (relpos > 0) goto out_of_frame; + + /* + * RPR cares about frame head pos. Need to call + * update_frameheadpos + */ + update_frameheadpos(winstate); + update_frametailpos(winstate); abs_pos = winstate->frametailpos - 1 + relpos; @@ -3613,6 +4007,14 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno, mark_pos = 0; /* keep compiler quiet */ break; } + + num_reduced_frame = row_is_in_reduced_frame(winobj, + winstate->frameheadpos + relpos); + if (num_reduced_frame < 0) + goto out_of_frame; + else if (num_reduced_frame > 0) + abs_pos = winstate->frameheadpos + relpos + + num_reduced_frame - 1; break; default: elog(ERROR, "unrecognized window seek type: %d", seektype); @@ -3631,15 +4033,13 @@ WinGetFuncArgInFrame(WindowObject winobj, int argno, *isout = false; if (set_mark) WinSetMarkPosition(winobj, mark_pos); - econtext->ecxt_outertuple = slot; - return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno), - econtext, isnull); + return 0; out_of_frame: if (isout) *isout = true; *isnull = true; - return (Datum) 0; + return -1; } /* @@ -3670,3 +4070,1430 @@ WinGetFuncArgCurrent(WindowObject winobj, int argno, bool *isnull) return ExecEvalExpr((ExprState *) list_nth(winobj->argstates, argno), econtext, isnull); } + +/* + * rpr_is_defined + * return true if Row pattern recognition is defined. + */ +static +bool +rpr_is_defined(WindowAggState *winstate) +{ + return winstate->patternVariableList != NIL; +} + +/* + * ----------------- + * row_is_in_reduced_frame + * Determine whether a row is in the current row's reduced window frame + * according to row pattern matching + * + * The row must has been already determined that it is in a full window frame + * and fetched it into slot. + * + * Returns: + * = 0, RPR is not defined. + * >0, if the row is the first in the reduced frame. Return the number of rows + * in the reduced frame. + * -1, if the row is unmatched row + * -2, if the row is in the reduced frame but needed to be skipped because of + * AFTER MATCH SKIP PAST LAST ROW + * ----------------- + */ +static +int +row_is_in_reduced_frame(WindowObject winobj, int64 pos) +{ + WindowAggState *winstate = winobj->winstate; + int state; + int rtn; + + if (!rpr_is_defined(winstate)) + { + /* + * RPR is not defined. Assume that we are always in the the reduced + * window frame. + */ + rtn = 0; +#ifdef RPR_DEBUG + elog(DEBUG1, "row_is_in_reduced_frame returns %d: pos: " INT64_FORMAT, + rtn, pos); +#endif + return rtn; + } + + state = get_reduced_frame_map(winstate, pos); + + if (state == RF_NOT_DETERMINED) + { + update_frameheadpos(winstate); + update_reduced_frame(winobj, pos); + } + + state = get_reduced_frame_map(winstate, pos); + + switch (state) + { + int64 i; + int num_reduced_rows; + + case RF_FRAME_HEAD: + num_reduced_rows = 1; + for (i = pos + 1; + get_reduced_frame_map(winstate, i) == RF_SKIPPED; i++) + num_reduced_rows++; + rtn = num_reduced_rows; + break; + + case RF_SKIPPED: + rtn = -2; + break; + + case RF_UNMATCHED: + rtn = -1; + break; + + default: + elog(ERROR, "Unrecognized state: %d at: " INT64_FORMAT, + state, pos); + break; + } + +#ifdef RPR_DEBUG + elog(DEBUG1, "row_is_in_reduced_frame returns %d: pos: " INT64_FORMAT, + rtn, pos); +#endif + return rtn; +} + +#define REDUCED_FRAME_MAP_INIT_SIZE 1024L + +/* + * create_reduced_frame_map + * Create reduced frame map + */ +static +void +create_reduced_frame_map(WindowAggState *winstate) +{ + winstate->reduced_frame_map = + MemoryContextAlloc(winstate->partcontext, + REDUCED_FRAME_MAP_INIT_SIZE); + winstate->alloc_sz = REDUCED_FRAME_MAP_INIT_SIZE; + clear_reduced_frame_map(winstate); +} + +/* + * clear_reduced_frame_map + * Clear reduced frame map + */ +static +void +clear_reduced_frame_map(WindowAggState *winstate) +{ + Assert(winstate->reduced_frame_map != NULL); + MemSet(winstate->reduced_frame_map, RF_NOT_DETERMINED, + winstate->alloc_sz); +} + +/* + * get_reduced_frame_map + * Get reduced frame map specified by pos + */ +static +int +get_reduced_frame_map(WindowAggState *winstate, int64 pos) +{ + Assert(winstate->reduced_frame_map != NULL); + Assert(pos >= 0); + + /* + * If pos is not in the reduced frame map, it means that any info + * regarding the pos has not been registered yet. So we return + * RF_NOT_DETERMINED. + */ + if (pos >= winstate->alloc_sz) + return RF_NOT_DETERMINED; + + return winstate->reduced_frame_map[pos]; +} + +/* + * register_reduced_frame_map + * Add/replace reduced frame map member at pos. + * If there's no enough space, expand the map. + */ +static +void +register_reduced_frame_map(WindowAggState *winstate, int64 pos, int val) +{ + int64 realloc_sz; + + Assert(winstate->reduced_frame_map != NULL); + + if (pos < 0) + elog(ERROR, "wrong pos: " INT64_FORMAT, pos); + + if (pos > winstate->alloc_sz - 1) + { + realloc_sz = winstate->alloc_sz * 2; + + winstate->reduced_frame_map = + repalloc(winstate->reduced_frame_map, realloc_sz); + + MemSet(winstate->reduced_frame_map + winstate->alloc_sz, + RF_NOT_DETERMINED, realloc_sz - winstate->alloc_sz); + + winstate->alloc_sz = realloc_sz; + } + + winstate->reduced_frame_map[pos] = val; +} + +/* + * update_reduced_frame + * Update reduced frame info. + */ +static +void +update_reduced_frame(WindowObject winobj, int64 pos) +{ + WindowAggState *winstate = winobj->winstate; + ListCell *lc1, + *lc2; + bool expression_result; + int num_matched_rows; + int64 original_pos; + bool anymatch; + StringInfo encoded_str; + StringSet *str_set; + bool greedy = false; + int64 result_pos, + i; + + /* + * Set of pattern variables evaluated to true. Each character corresponds + * to pattern variable. Example: str_set[0] = "AB"; str_set[1] = "AC"; In + * this case at row 0 A and B are true, and A and C are true in row 1. + */ + + /* initialize pattern variables set */ + str_set = string_set_init(); + + /* save original pos */ + original_pos = pos; + + /* + * Check if the pattern does not include any greedy quantifier. If it does + * not, we can just apply the pattern to each row. If it succeeds, we are + * done. + */ + foreach(lc1, winstate->patternRegexpList) + { + char *quantifier = strVal(lfirst(lc1)); + + if (*quantifier == '+' || *quantifier == '*') + { + greedy = true; + break; + } + } + + /* + * Non greedy case + */ + if (!greedy) + { + num_matched_rows = 0; + + foreach(lc1, winstate->patternVariableList) + { + char *vname = strVal(lfirst(lc1)); + + encoded_str = makeStringInfo(); + +#ifdef RPR_DEBUG + elog(DEBUG1, "pos: " INT64_FORMAT " pattern vname: %s", + pos, vname); +#endif + expression_result = false; + + /* evaluate row pattern against current row */ + result_pos = evaluate_pattern(winobj, pos, vname, + encoded_str, &expression_result); + destroyStringInfo(encoded_str); + + if (!expression_result || result_pos < 0) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "expression result is false or out of frame"); +#endif + register_reduced_frame_map(winstate, original_pos, + RF_UNMATCHED); + return; + } + /* move to next row */ + pos++; + num_matched_rows++; + } +#ifdef RPR_DEBUG + elog(DEBUG1, "pattern matched"); +#endif + register_reduced_frame_map(winstate, original_pos, RF_FRAME_HEAD); + + for (i = original_pos + 1; i < original_pos + num_matched_rows; i++) + { + register_reduced_frame_map(winstate, i, RF_SKIPPED); + } + return; + } + + /* + * Greedy quantifiers included. Loop over until none of pattern matches or + * encounters end of frame. + */ + for (;;) + { + int init_size; + + result_pos = -1; + + /* + * Loop over each PATTERN variable. + */ + anymatch = false; + + /* + * makeStringInfo creates initial data size to be 1024 bytes, which is + * too large for us because we only need the initial data as the + * number PATTERN variables (+null terminate), which is usually less + * than 10 bytes. So we reallocate the initial data size as small as + * the number of PATTERN variables. + */ + encoded_str = makeStringInfo(); + pfree(encoded_str->data); + init_size = list_length(winstate->patternVariableList) + 1; + encoded_str->data = (char *) palloc0(init_size); + encoded_str->maxlen = init_size; + + forboth(lc1, winstate->patternVariableList, lc2, + winstate->patternRegexpList) + { + char *vname = strVal(lfirst(lc1)); +#ifdef RPR_DEBUG + char *quantifier = strVal(lfirst(lc2)); + + elog(DEBUG1, "pos: " INT64_FORMAT " pattern vname: %s quantifier: %s", + pos, vname, quantifier); +#endif + expression_result = false; + + /* evaluate row pattern against current row */ + result_pos = evaluate_pattern(winobj, pos, vname, + encoded_str, &expression_result); + if (expression_result) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "expression result is true"); +#endif + anymatch = true; + } + + /* + * If out of frame, we are done. + */ + if (result_pos < 0) + break; + } + + if (!anymatch) + { + /* none of patterns matched. */ + break; + } + + string_set_add(str_set, encoded_str, 0); + +#ifdef RPR_DEBUG + elog(DEBUG1, "pos: " INT64_FORMAT " encoded_str: %s", + encoded_str->data); +#endif + + /* move to next row */ + pos++; + + if (result_pos < 0) + { + /* out of frame */ + break; + } + } + + if (string_set_get_size(str_set) == 0) + { + /* no match found in the first row */ + register_reduced_frame_map(winstate, original_pos, RF_UNMATCHED); + destroyStringInfo(encoded_str); + return; + } + +#ifdef RPR_DEBUG + elog(DEBUG2, "pos: " INT64_FORMAT " encoded_str: %s", + pos, encoded_str->data); +#endif + + /* look for matching pattern variable sequence */ +#ifdef RPR_DEBUG + elog(DEBUG1, "search_str_set started"); +#endif + num_matched_rows = search_str_set(winstate, str_set); + +#ifdef RPR_DEBUG + elog(DEBUG1, "search_str_set returns: %d", num_matched_rows); +#endif + string_set_discard(str_set); + + /* + * We are at the first row in the reduced frame. Save the number of + * matched rows as the number of rows in the reduced frame. + */ + if (num_matched_rows <= 0) + { + /* no match */ + register_reduced_frame_map(winstate, original_pos, RF_UNMATCHED); + } + else + { + register_reduced_frame_map(winstate, original_pos, RF_FRAME_HEAD); + + for (i = original_pos + 1; i < original_pos + num_matched_rows; i++) + { + register_reduced_frame_map(winstate, i, RF_SKIPPED); + } + } + + return; +} + +/* + * search_str_set + * + * Perform pattern matching using "pattern" against input_str_set. pattern is + * a regular expression string derived from PATTERN clause. Note that the + * regular expression string is prefixed by '^' and followed by initials + * represented in a same way as str_set. str_set is a set of StringInfo. Each + * StringInfo has a string comprising initials of pattern variable strings + * being true in a row. The initials are one of [a-z], parallel to the order + * of variable names in DEFINE clause. Suppose DEFINE has variables START, UP + * and DOWN. If PATTERN has START, UP+ and DOWN, then the initials in PATTERN + * will be 'a', 'b' and 'c'. The "pattern" will be "^ab+c". + * + * variable_pos is an array representing the order of pattern variable string + * initials in PATTERN clause. For example initial 'a' potion is in + * variable_pos[0].pos[0] = 0. Note that if the pattern is "START UP DOWN UP" + * (UP appears twice), then "UP" (initial is 'b') has two position 1 and + * 3. Thus variable_pos for b is variable_pos[1].pos[0] = 1 and + * variable_pos[1].pos[1] = 3. + * + * Returns the longest number of the matching rows (greedy matching) if + * quatifier '+' or '*' is included in "pattern". + */ +static +int +search_str_set(WindowAggState *winstate, StringSet *input_str_set) +{ + char *pattern; /* search regexp pattern */ + VariablePos *variable_pos; + int set_size; /* number of rows in the set */ + int resultlen; + int index; + StringSet *new_str_set; + int new_str_size; + int len; + int info; + char tail_pattern_initial; + + /* + * Set last initial char to tail_pattern_initial if we can apply "tail + * pattern initial optimization". If the last regexp component in pattern + * is with '+' quatifier, set the initial to tail_pattern_initial. For + * example if pattern = "ab+", tail_pattern_initial will be 'b'. + * Otherwise, tail_pattern_initial is '\0'. + */ + pattern = winstate->pattern_str->data; + if (pattern[strlen(pattern) - 1] == '+') + tail_pattern_initial = pattern[strlen(pattern) - 2]; + else + tail_pattern_initial = '\0'; + + /* + * Generate all possible pattern variable name initials as a set of + * StringInfo named "new_str_set". For example, if we have two rows + * having "ab" (row 0) and "ac" (row 1) in the input str_set, new_str_set + * will have set of StringInfo "aa", "ac", "ba" and "bc" in the end. + */ + variable_pos = winstate->variable_pos; + new_str_set = generate_patterns(input_str_set, pattern, variable_pos, + tail_pattern_initial); + + /* + * Perform pattern matching to find out the longest match. + */ + new_str_size = string_set_get_size(new_str_set); + len = 0; + resultlen = 0; + set_size = string_set_get_size(input_str_set); + + for (index = 0; index < new_str_size; index++) + { + StringInfo s; + + s = string_set_get(new_str_set, index, &info); + if (s == NULL) + continue; /* no data */ + + /* + * If the string is scheduled to be discarded, we just disregard it. + */ + if (info & STRSET_DISCARDED) + continue; + + len = do_pattern_match(pattern, s->data, s->len); + if (len > resultlen) + { + /* remember the longest match */ + resultlen = len; + + /* + * If the size of result set is equal to the number of rows in the + * set, we are done because it's not possible that the number of + * matching rows exceeds the number of rows in the set. + */ + if (resultlen >= set_size) + break; + } + } + + /* we no longer need new string set */ + string_set_discard(new_str_set); + + return resultlen; +} + +/* + * generate_patterns + * + * Generate all possible pattern variable name initials in 'input_str_set' as + * a set of StringInfo and return it. For example, if we have two rows having + * "ab" (row 0) and "ac" (row 1) in 'input str_set', returned StringSet will + * have set of StringInfo "aa", "ac", "ba" and "bc" in the end. + * 'variable_pos' and 'tail_pattern_initial' are used for pruning + * optimization. + */ +static +StringSet * +generate_patterns(StringSet *input_str_set, char *pattern, + VariablePos *variable_pos, char tail_pattern_initial) +{ + StringSet *old_str_set, + *new_str_set; + int index; + int set_size; + int old_set_size; + int info; + int resultlen; + StringInfo str; + int i; + char *p; + + new_str_set = string_set_init(); + set_size = string_set_get_size(input_str_set); + if (set_size == 0) /* if there's no row in input, return empty + * set */ + return new_str_set; + + resultlen = 0; + + /* + * Generate initial new_string_set for input row 0. + */ + str = string_set_get(input_str_set, 0, &info); + p = str->data; + + /* + * Loop over each new pattern variable char. + */ + while (*p) + { + StringInfo new = makeStringInfo(); + + /* add pattern variable char */ + appendStringInfoChar(new, *p); + /* add new one to string set */ + string_set_add(new_str_set, new, 0); + p++; /* next pattern variable */ + } + + /* + * Generate new_string_set for each input row. + */ + for (index = 1; index < set_size; index++) + { + /* previous new str set now becomes old str set */ + old_str_set = new_str_set; + new_str_set = string_set_init(); /* create new string set */ + /* pick up input string */ + str = string_set_get(input_str_set, index, &info); + old_set_size = string_set_get_size(old_str_set); + + /* + * Loop over each row in the previous result set. + */ + for (i = 0; i < old_set_size; i++) + { + char last_old_char; + int old_str_len; + int old_info; + StringInfo old; + + old = string_set_get(old_str_set, i, &old_info); + p = old->data; + old_str_len = old->len; + if (old_str_len > 0) + last_old_char = p[old_str_len - 1]; + else + last_old_char = '\0'; + + /* Can this old set be discarded? */ + if (old_info & STRSET_DISCARDED) + continue; /* discard the old string */ + + /* Is this old set frozen? */ + else if (old_info & STRSET_FROZEN) + { + /* if shorter match. we can discard it */ + if (old_str_len < resultlen) + continue; /* discard the shorter string */ + + /* move the old set to new_str_set */ + string_set_add(new_str_set, old, old_info); + old_str_set->str_set[i] = NULL; + continue; + } + + /* + * loop over each pattern variable initial char in the input set. + */ + for (p = str->data; *p; p++) + { + /* + * Optimization. Check if the row's pattern variable initial + * character position is greater than or equal to the old + * set's last pattern variable initial character position. For + * example, if the old set's last pattern variable initials + * are "ab", then the new pattern variable initial can be "b" + * or "c" but can not be "a", if the initials in PATTERN is + * something like "a b c" or "a b+ c+" etc. This optimization + * is possible when we only allow "+" quantifier. + */ + if (variable_pos_compare(variable_pos, last_old_char, *p)) + + /* + * Satisfied the condition. Add new pattern char to + * new_str_set if it looks good. + */ + resultlen = add_pattern(old, old_info, new_str_set, *p, + pattern, tail_pattern_initial, resultlen); + else + + /* + * The old_str did not satisfy the condition and it cannot + * be extended further. "Freeze" it. + */ + resultlen = freeze_pattern(old, old_info, + new_str_set, pattern, resultlen); + } + } + /* we no longer need old string set */ + string_set_discard(old_str_set); + } + return new_str_set; +} + +/* + * add_pattern + * + * Make a copy of 'old' (along with 'old_info' flag) and add new pattern char + * 'c' to it. Then add it to 'new_str_set'. 'pattern' and + * 'tail_pattern_initial' is checked to determine whether the copy is worth to + * add to new_str_set or not. The match length (possibly longer than + * 'resultlen') is returned. + */ +static +int +add_pattern(StringInfo old, int old_info, StringSet *new_str_set, char c, + char *pattern, char tail_pattern_initial, int resultlen) +{ + StringInfo new; + int info; + int len; + + /* + * New char in the input row satisfies the condition above. + */ + new = makeStringInfo(); /* copy source string */ + enlargeStringInfo(new, old->len + 1); + appendStringInfoString(new, old->data); + + /* add pattern variable char */ + appendStringInfoChar(new, c); + + /* + * Adhoc optimization. If the first letter in the input string is in the + * head and second position and there's no associated quatifier '+', then + * we can dicard the input because there's no chance to expand the string + * further. + * + * For example, pattern "abc" cannot match "aa". + */ + if (pattern[1] == new->data[0] && + pattern[1] == new->data[1] && + pattern[2] != '+' && + pattern[1] != pattern[2]) + { + destroyStringInfo(new); + return resultlen; + } + + info = old_info; + + /* + * Check if we can apply "tail pattern initial optimization". If the last + * regexp component in pattern has '+' quantifier, the component is set to + * the last pattern initial. For example if pattern is "ab+", + * tail_pattern_initial will become 'b'. Otherwise, tail_pattern_initial + * is '\0'. If the tail pattern initial optimization is possible, we do + * not need to apply regular expression match again. Suppose we have the + * previous string ended with "b" and the it was confirmed the regular + * expression match, then char 'b' can be added to the string without + * applying the regular expression match again. + */ + if (c == tail_pattern_initial) /* tail pattern initial optimization + * possible? */ + { + /* + * Is already confirmed to be matched with pattern? + */ + if ((info & STRSET_MATCHED) == 0) + { + /* not confirmed yet */ + len = do_pattern_match(pattern, new->data, new->len); + if (len > 0) + info = STRSET_MATCHED; /* set already confirmed flag */ + } + else + + /* + * already confirmed. Use the string length as the matching length + */ + len = new->len; + + /* update the longest match length if needed */ + if (len > resultlen) + resultlen = len; + } + + /* add new StringInfo to the string set */ + string_set_add(new_str_set, new, info); + + return resultlen; +} + +/* + * freeze_pattern + * + * "Freeze" 'old' (along with 'old_info' flag) and add it to + * 'new_str_set'. Frozen string is known to not be expanded further. The frozen + * string is check if it satisfies 'pattern'. If it does not, "discarded" + * mark is added. The discarded mark is also added if the match length is + * shorter than the current longest match length. The match length (possibly + * longer than 'resultlen') is returned. + */ +static +int +freeze_pattern(StringInfo old, int old_info, StringSet *new_str_set, + char *pattern, int resultlen) +{ + int len; + StringInfo new; + int new_str_size; + int new_index; + + /* + * We are freezing this pattern string. If the pattern string length is + * shorter than the current longest string length, we don't need to keep + * it. + */ + if (old->len < resultlen) + return resultlen; + + if (old_info & STRSET_MATCHED) + /* we don't need to apply pattern match again */ + len = old->len; + else + { + /* apply pattern match */ + len = do_pattern_match(pattern, old->data, old->len); + if (len <= 0) + { + /* no match. we can discard it */ + return resultlen; + } + } + if (len < resultlen) + { + /* shorter match. we can discard it */ + return resultlen; + } + + /* + * Match length is the longest so far + */ + resultlen = len; /* remember the longest match */ + + /* freeze the pattern string */ + new = makeStringInfo(); + enlargeStringInfo(new, old->len + 1); + appendStringInfoString(new, old->data); + /* set frozen mark */ + string_set_add(new_str_set, new, STRSET_FROZEN); + + /* + * Search new_str_set to find out frozen entries that have shorter match + * length. Mark them as "discard" so that they are discarded in the next + * round. + */ + new_str_size = + string_set_get_size(new_str_set) - 1; + + /* loop over new_str_set */ + for (new_index = 0; new_index < new_str_size; new_index++) + { + int info; + + new = string_set_get(new_str_set, new_index, &info); + + /* + * If this is frozen and is not longer than the current longest match + * length, we don't need to keep this. + */ + if (info & STRSET_FROZEN && new->len < resultlen) + { + /* + * mark this set to discard in the next round + */ + info |= STRSET_DISCARDED; + new_str_set->info[new_index] = info; + } + } + return resultlen; +} + +/* + * do_pattern_match + * + * Perform pattern match using 'pattern' against 'encoded_str' whose length is + * 'len' bytes (without null terminate). Returns matching number of rows if + * matching is succeeded. Otherwise returns 0. + */ +static +int +do_pattern_match(char *pattern, char *encoded_str, int len) +{ + static regex_t *regcache = NULL; + static regex_t preg; + static char patbuf[1024]; /* most recent 'pattern' is cached here */ + int plen; + int cflags = REG_EXTENDED; + size_t nmatch = 1; + int eflags = 0; + regmatch_t pmatch[1]; + int sts; + pg_wchar *data; + int data_len; + + /* + * Compile regexp if cache does not exist or existing cache is not same as + * "pattern". + */ + if (strcmp(patbuf, pattern)) + { + /* + * The compiled re must live in top memory context because patbuf is + * static data. + */ + MemoryContext oldContext = MemoryContextSwitchTo(TopMemoryContext); + + if (regcache != NULL) + pg_regfree(regcache); /* free previous re */ + + /* we need to convert to char to pg_wchar */ + plen = strlen(pattern); + data = (pg_wchar *) palloc((plen + 1) * sizeof(pg_wchar)); + data_len = pg_mb2wchar_with_len(pattern, data, plen); + /* compile re */ + sts = pg_regcomp(&preg, /* compiled re */ + data, /* target pattern */ + data_len, /* length of pattern */ + cflags, /* compile option */ + C_COLLATION_OID /* collation */ + ); + pfree(data); + + MemoryContextSwitchTo(oldContext); + + if (sts != REG_OKAY) + { + /* re didn't compile (no need for pg_regfree, if so) */ + ereport(ERROR, + (errcode(ERRCODE_INVALID_REGULAR_EXPRESSION), + errmsg("invalid regular expression: %s", pattern))); + } + + /* save cache */ + regcache = &preg; + strncpy(patbuf, pattern, sizeof(patbuf)); + } + + data = (pg_wchar *) palloc((len + 1) * sizeof(pg_wchar)); + data_len = pg_mb2wchar_with_len(encoded_str, data, len); + + /* execute the regular expression match */ + sts = pg_regexec( + &preg, /* compiled re */ + data, /* target string */ + data_len, /* length of encoded_str */ + 0, /* search start */ + NULL, /* rm details */ + nmatch, /* number of match sub re */ + pmatch, /* match result details */ + eflags); + + pfree(data); + + if (sts != REG_OKAY) + { + if (sts != REG_NOMATCH) + { + char errMsg[100]; + + pg_regerror(sts, &preg, errMsg, sizeof(errMsg)); + ereport(ERROR, + (errcode(ERRCODE_INVALID_REGULAR_EXPRESSION), + errmsg("regular expression failed: %s", errMsg))); + } + return 0; /* does not match */ + } + + len = pmatch[0].rm_eo; /* return match length */ + return len; + +} + +/* + * evaluate_pattern + * + * Evaluate expression associated with PATTERN variable vname. current_pos is + * relative row position in a frame (starting from 0). If vname is evaluated + * to true, initial letters associated with vname is appended to + * encode_str. result is out paramater representing the expression evaluation + * result is true of false. + *--------- + * Return values are: + * >=0: the last match absolute row position + * otherwise out of frame. + *--------- + */ +static +int64 +evaluate_pattern(WindowObject winobj, int64 current_pos, + char *vname, StringInfo encoded_str, bool *result) +{ + WindowAggState *winstate = winobj->winstate; + ExprContext *econtext = winstate->ss.ps.ps_ExprContext; + ListCell *lc1, + *lc2, + *lc3; + ExprState *pat; + Datum eval_result; + bool out_of_frame = false; + bool isnull; + TupleTableSlot *slot; + + forthree(lc1, winstate->defineVariableList, + lc2, winstate->defineClauseList, + lc3, winstate->defineInitial) + { + char initial; /* initial letter associated with vname */ + char *name = strVal(lfirst(lc1)); + + if (strcmp(vname, name)) + continue; + + initial = *(strVal(lfirst(lc3))); + + /* set expression to evaluate */ + pat = lfirst(lc2); + + /* get current, previous and next tuples */ + if (!get_slots(winobj, current_pos)) + { + out_of_frame = true; + } + else + { + /* evaluate the expression */ + eval_result = ExecEvalExpr(pat, econtext, &isnull); + if (isnull) + { + /* expression is NULL */ +#ifdef RPR_DEBUG + elog(DEBUG1, "expression for %s is NULL at row: " INT64_FORMAT, + vname, current_pos); +#endif + *result = false; + } + else + { + if (!DatumGetBool(eval_result)) + { + /* expression is false */ +#ifdef RPR_DEBUG + elog(DEBUG1, "expression for %s is false at row: " INT64_FORMAT, + vname, current_pos); +#endif + *result = false; + } + else + { + /* expression is true */ +#ifdef RPR_DEBUG + elog(DEBUG1, "expression for %s is true at row: " INT64_FORMAT, + vname, current_pos); +#endif + appendStringInfoChar(encoded_str, initial); + *result = true; + } + } + + slot = winstate->temp_slot_1; + if (slot != winstate->null_slot) + ExecClearTuple(slot); + slot = winstate->prev_slot; + if (slot != winstate->null_slot) + ExecClearTuple(slot); + slot = winstate->next_slot; + if (slot != winstate->null_slot) + ExecClearTuple(slot); + + break; + } + + if (out_of_frame) + { + *result = false; + return -1; + } + } + return current_pos; +} + +/* + * get_slots + * + * Get current, previous and next tuples. + * Returns false if current row is out of partition/full frame. + */ +static +bool +get_slots(WindowObject winobj, int64 current_pos) +{ + WindowAggState *winstate = winobj->winstate; + TupleTableSlot *slot; + int ret; + ExprContext *econtext; + + econtext = winstate->ss.ps.ps_ExprContext; + + /* set up current row tuple slot */ + slot = winstate->temp_slot_1; + if (!window_gettupleslot(winobj, current_pos, slot)) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "current row is out of partition at:" INT64_FORMAT, + current_pos); +#endif + return false; + } + ret = row_is_in_frame(winstate, current_pos, slot); + if (ret <= 0) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "current row is out of frame at: " INT64_FORMAT, + current_pos); +#endif + ExecClearTuple(slot); + return false; + } + econtext->ecxt_outertuple = slot; + + /* for PREV */ + if (current_pos > 0) + { + slot = winstate->prev_slot; + if (!window_gettupleslot(winobj, current_pos - 1, slot)) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "previous row is out of partition at: " INT64_FORMAT, + current_pos - 1); +#endif + econtext->ecxt_scantuple = winstate->null_slot; + } + else + { + ret = row_is_in_frame(winstate, current_pos - 1, slot); + if (ret <= 0) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "previous row is out of frame at: " INT64_FORMAT, + current_pos - 1); +#endif + ExecClearTuple(slot); + econtext->ecxt_scantuple = winstate->null_slot; + } + else + { + econtext->ecxt_scantuple = slot; + } + } + } + else + econtext->ecxt_scantuple = winstate->null_slot; + + /* for NEXT */ + slot = winstate->next_slot; + if (!window_gettupleslot(winobj, current_pos + 1, slot)) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "next row is out of partiton at: " INT64_FORMAT, + current_pos + 1); +#endif + econtext->ecxt_innertuple = winstate->null_slot; + } + else + { + ret = row_is_in_frame(winstate, current_pos + 1, slot); + if (ret <= 0) + { +#ifdef RPR_DEBUG + elog(DEBUG1, "next row is out of frame at: " INT64_FORMAT, + current_pos + 1); +#endif + ExecClearTuple(slot); + econtext->ecxt_innertuple = winstate->null_slot; + } + else + econtext->ecxt_innertuple = slot; + } + return true; +} + +/* + * pattern_initial + * + * Return pattern variable initial character + * matching with pattern variable name vname. + * If not found, return 0. + */ +static +char +pattern_initial(WindowAggState *winstate, char *vname) +{ + char initial; + char *name; + ListCell *lc1, + *lc2; + + forboth(lc1, winstate->defineVariableList, + lc2, winstate->defineInitial) + { + name = strVal(lfirst(lc1)); /* DEFINE variable name */ + initial = *(strVal(lfirst(lc2))); /* DEFINE variable initial */ + + + if (!strcmp(name, vname)) + return initial; /* found */ + } + return 0; +} + +/* + * string_set_init + * + * Create dynamic set of StringInfo. + */ +static +StringSet * +string_set_init(void) +{ +/* Initial allocation size of str_set */ +#define STRING_SET_ALLOC_SIZE 1024 + + StringSet *string_set; + Size set_size; + + string_set = palloc0(sizeof(StringSet)); + string_set->set_index = 0; + set_size = STRING_SET_ALLOC_SIZE; + string_set->str_set = palloc(set_size * sizeof(StringInfo)); + string_set->info = palloc0(set_size * sizeof(int)); + string_set->set_size = set_size; + + return string_set; +} + +/* + * string_set_add + * + * Add StringInfo str to StringSet string_set. + */ +static +void +string_set_add(StringSet *string_set, StringInfo str, int info) +{ + Size set_size; + Size old_set_size; + + set_size = string_set->set_size; + if (string_set->set_index >= set_size) + { + old_set_size = set_size; + set_size *= 2; + string_set->str_set = repalloc(string_set->str_set, + set_size * sizeof(StringInfo)); + string_set->info = repalloc0(string_set->info, + old_set_size * sizeof(int), + set_size * sizeof(int)); + string_set->set_size = set_size; + } + + string_set->info[string_set->set_index] = info; + string_set->str_set[string_set->set_index++] = str; + + return; +} + +/* + * string_set_get + * + * Returns StringInfo specified by index. + * If there's no data yet, returns NULL. + */ +static +StringInfo +string_set_get(StringSet *string_set, int index, int *info) +{ + *info = 0; + + /* no data? */ + if (index == 0 && string_set->set_index == 0) + return NULL; + + if (index < 0 || index >= string_set->set_index) + elog(ERROR, "invalid index: %d", index); + + *info = string_set->info[index]; + + return string_set->str_set[index]; +} + +/* + * string_set_get_size + * + * Returns the size of StringSet. + */ +static +int +string_set_get_size(StringSet *string_set) +{ + return string_set->set_index; +} + +/* + * string_set_discard + * Discard StringSet. + * All memory including StringSet itself is freed. + */ +static +void +string_set_discard(StringSet *string_set) +{ + int i; + + for (i = 0; i < string_set->set_index; i++) + { + StringInfo str = string_set->str_set[i]; + + if (str) + destroyStringInfo(str); + } + pfree(string_set->info); + pfree(string_set->str_set); + pfree(string_set); +} + +/* + * variable_pos_init + * + * Create and initialize variable postion structure + */ +static +VariablePos * +variable_pos_init(void) +{ + VariablePos *variable_pos; + + variable_pos = palloc(sizeof(VariablePos) * NUM_ALPHABETS); + MemSet(variable_pos, -1, sizeof(VariablePos) * NUM_ALPHABETS); + return variable_pos; +} + +/* + * variable_pos_register + * + * Register pattern variable whose initial is initial into postion index. + * pos is position of initial. + * If pos is already registered, register it at next empty slot. + */ +static +void +variable_pos_register(VariablePos *variable_pos, char initial, int pos) +{ + int index = initial - 'a'; + int slot; + int i; + + if (pos < 0 || pos > NUM_ALPHABETS) + elog(ERROR, "initial is not valid char: %c", initial); + + for (i = 0; i < NUM_ALPHABETS; i++) + { + slot = variable_pos[index].pos[i]; + if (slot < 0) + { + /* empty slot found */ + variable_pos[index].pos[i] = pos; + return; + } + } + elog(ERROR, "no empty slot for initial: %c", initial); +} + +/* + * variable_pos_compare + * + * Returns true if initial1 can be followed by initial2 + */ +static +bool +variable_pos_compare(VariablePos *variable_pos, char initial1, char initial2) +{ + int index1, + index2; + int pos1, + pos2; + + for (index1 = 0;; index1++) + { + pos1 = variable_pos_fetch(variable_pos, initial1, index1); + if (pos1 < 0) + break; + + for (index2 = 0;; index2++) + { + pos2 = variable_pos_fetch(variable_pos, initial2, index2); + if (pos2 < 0) + break; + if (pos1 <= pos2) + return true; + } + } + return false; +} + +/* + * variable_pos_fetch + * + * Fetch position of pattern variable whose initial is initial, and whose index + * is index. If no postion was registered by initial, index, returns -1. + */ +static +int +variable_pos_fetch(VariablePos *variable_pos, char initial, int index) +{ + int pos = initial - 'a'; + + if (pos < 0 || pos > NUM_ALPHABETS) + elog(ERROR, "initial is not valid char: %c", initial); + + if (index < 0 || index > NUM_ALPHABETS) + elog(ERROR, "index is not valid: %d", index); + + return variable_pos[pos].pos[index]; +} + +/* + * variable_pos_build + * + * Build VariablePos structure and return it. + */ +static +VariablePos * +variable_pos_build(WindowAggState *winstate) +{ + VariablePos *variable_pos; + StringInfo pattern_str; + int initial_index = 0; + ListCell *lc1, + *lc2; + + variable_pos = winstate->variable_pos = variable_pos_init(); + pattern_str = winstate->pattern_str = makeStringInfo(); + appendStringInfoChar(pattern_str, '^'); + + forboth(lc1, winstate->patternVariableList, + lc2, winstate->patternRegexpList) + { + char *vname = strVal(lfirst(lc1)); + char *quantifier = strVal(lfirst(lc2)); + char initial; + + initial = pattern_initial(winstate, vname); + Assert(initial != 0); + appendStringInfoChar(pattern_str, initial); + if (quantifier[0]) + appendStringInfoChar(pattern_str, quantifier[0]); + + /* + * Register the initial at initial_index. If the initial appears more + * than once, all of it's initial_index will be recorded. This could + * happen if a pattern variable appears in the PATTERN clause more + * than once like "UP DOWN UP" "UP UP UP". + */ + variable_pos_register(variable_pos, initial, initial_index); + + initial_index++; + } + + return variable_pos; +} diff --git a/src/backend/utils/adt/windowfuncs.c b/src/backend/utils/adt/windowfuncs.c index 473c61569f..3142a8bc06 100644 --- a/src/backend/utils/adt/windowfuncs.c +++ b/src/backend/utils/adt/windowfuncs.c @@ -13,6 +13,9 @@ */ #include "postgres.h" +#include "catalog/pg_collation_d.h" +#include "executor/executor.h" +#include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "nodes/supportnodes.h" #include "utils/fmgrprotos.h" @@ -37,11 +40,19 @@ typedef struct int64 remainder; /* (total rows) % (bucket num) */ } ntile_context; +/* + * rpr process information. + * Used for AFTER MATCH SKIP PAST LAST ROW + */ +typedef struct SkipContext +{ + int64 pos; /* last row absolute position */ +} SkipContext; + static bool rank_up(WindowObject winobj); static Datum leadlag_common(FunctionCallInfo fcinfo, bool forward, bool withoffset, bool withdefault); - /* * utility routine for *_rank functions. */ @@ -674,7 +685,7 @@ window_last_value(PG_FUNCTION_ARGS) bool isnull; result = WinGetFuncArgInFrame(winobj, 0, - 0, WINDOW_SEEK_TAIL, true, + 0, WINDOW_SEEK_TAIL, false, &isnull, NULL); if (isnull) PG_RETURN_NULL(); @@ -714,3 +725,25 @@ window_nth_value(PG_FUNCTION_ARGS) PG_RETURN_DATUM(result); } + +/* + * prev + * Dummy function to invoke RPR's navigation operator "PREV". + * This is *not* a window function. + */ +Datum +window_prev(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(PG_GETARG_DATUM(0)); +} + +/* + * next + * Dummy function to invoke RPR's navigation operation "NEXT". + * This is *not* a window function. + */ +Datum +window_next(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(PG_GETARG_DATUM(0)); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 2dcc2d42da..ee11ee2c8b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10664,6 +10664,12 @@ { oid => '3114', descr => 'fetch the Nth row value', proname => 'nth_value', prokind => 'w', prorettype => 'anyelement', proargtypes => 'anyelement int4', prosrc => 'window_nth_value' }, +{ oid => '8126', descr => 'previous value', + proname => 'prev', provolatile => 's', prorettype => 'anyelement', + proargtypes => 'anyelement', prosrc => 'window_prev' }, +{ oid => '8127', descr => 'next value', + proname => 'next', provolatile => 's', prorettype => 'anyelement', + proargtypes => 'anyelement', prosrc => 'window_next' }, # functions for range types { oid => '3832', descr => 'I/O', diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 1590b64392..f96782fd6a 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -2585,6 +2585,37 @@ typedef enum WindowAggStatus * tuples during spool */ } WindowAggStatus; +#define RF_NOT_DETERMINED 0 +#define RF_FRAME_HEAD 1 +#define RF_SKIPPED 2 +#define RF_UNMATCHED 3 + +/* + * Allowed PATTERN variables positions. + * Used in RPR. + * + * pos represents the pattern variable defined order in DEFINE caluase. For + * example. "DEFINE START..., UP..., DOWN ..." and "PATTERN START UP DOWN UP" + * will create: + * VariablePos[0].pos[0] = 0; START + * VariablePos[1].pos[0] = 1; UP + * VariablePos[1].pos[1] = 3; UP + * VariablePos[2].pos[0] = 2; DOWN + * + * Note that UP has two pos because UP appears in PATTERN twice. + * + * By using this strucrture, we can know which pattern variable can be followed + * by which pattern variable(s). For example, START can be followed by UP and + * DOWN since START's pos is 0, and UP's pos is 1 or 3, DOWN's pos is 2. + * DOWN can be followed by UP since UP's pos is either 1 or 3. + * + */ +#define NUM_ALPHABETS 26 /* we allow [a-z] variable initials */ +typedef struct VariablePos +{ + int pos[NUM_ALPHABETS]; /* postions in PATTERN */ +} VariablePos; + typedef struct WindowAggState { ScanState ss; /* its first field is NodeTag */ @@ -2644,6 +2675,21 @@ typedef struct WindowAggState int64 groupheadpos; /* current row's peer group head position */ int64 grouptailpos; /* " " " " tail position (group end+1) */ + /* these fields are used in Row pattern recognition: */ + RPSkipTo rpSkipTo; /* Row Pattern Skip To type */ + List *patternVariableList; /* list of row pattern variables names + * (list of String) */ + List *patternRegexpList; /* list of row pattern regular expressions + * ('+' or ''. list of String) */ + List *defineVariableList; /* list of row pattern definition + * variables (list of String) */ + List *defineClauseList; /* expression for row pattern definition + * search conditions ExprState list */ + List *defineInitial; /* list of row pattern definition variable + * initials (list of String) */ + VariablePos *variable_pos; /* list of pattern variable positions */ + StringInfo pattern_str; /* PATTERN initials */ + MemoryContext partcontext; /* context for partition-lifespan data */ MemoryContext aggcontext; /* shared context for aggregate working data */ MemoryContext curaggcontext; /* current aggregate's working data */ @@ -2671,6 +2717,18 @@ typedef struct WindowAggState TupleTableSlot *agg_row_slot; TupleTableSlot *temp_slot_1; TupleTableSlot *temp_slot_2; + + /* temporary slots for RPR */ + TupleTableSlot *prev_slot; /* PREV row navigation operator */ + TupleTableSlot *next_slot; /* NEXT row navigation operator */ + TupleTableSlot *null_slot; /* all NULL slot */ + + /* + * Each byte corresponds to a row positioned at absolute its pos in + * partition. See above definition for RF_* + */ + char *reduced_frame_map; + int64 alloc_sz; /* size of the map */ } WindowAggState; /* ---------------- -- 2.25.1 ----Next_Part(Tue_Dec_31_08_57_07_2024_963)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v27-0006-Row-pattern-recognition-patch-docs.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
end of thread, other threads:[~2024-12-30 23:53 UTC | newest] Thread overview: 30+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-03-08 22:15 [PATCH v9 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-08 22:15 [PATCH v12 05/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-08 22:15 [PATCH v11 4/9] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-08 22:15 [PATCH v10 4/9] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-08 22:15 [PATCH v13 3/8] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v18 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v26 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v16 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v24 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v21 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v30 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v28 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v15 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v19 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v23 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v22 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v32 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v27 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v25 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v35 3/7] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v31 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v20 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v36 3/7] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v33 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v34 03/15] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v37 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v14 3/8] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-10 03:40 [PATCH v17 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <[email protected]> 2020-03-13 07:59 [PATCH v48 3/7] Make archiver process an auxiliary process Kyotaro Horiguchi <[email protected]> 2024-12-30 23:53 [PATCH v27 5/9] Row pattern recognition patch (executor). Tatsuo Ishii <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox