agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v9 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. 22+ messages / 2 participants [nested] [flat]
* [PATCH v9 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-08 22:15 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-08 22:15 UTC (permalink / raw) Need catversion bumped? --- src/backend/utils/adt/genfile.c | 51 ++++++++++++++++++++++++++------- src/include/catalog/pg_proc.dat | 6 ++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 897b11a77d..4699aea568 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -524,7 +524,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) /* Generic function to return a directory listing of files */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok, bool dir_ok) { FuncCallContext *funcctx; struct dirent *de; @@ -540,13 +540,17 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) fctx = palloc(sizeof(directory_fctx)); - tupdesc = CreateTemplateTupleDesc(3); + tupdesc = CreateTemplateTupleDesc(dir_ok ? 4:3); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "name", TEXTOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "size", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 3, "modification", TIMESTAMPTZOID, -1, 0); + if (dir_ok) + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "isdir", + BOOLOID, -1, 0); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); fctx->location = pstrdup(dir); @@ -575,8 +579,8 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; HeapTuple tuple; @@ -592,13 +596,21 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", path))); - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) + if (S_ISDIR(attrib.st_mode)) + { + if (!dir_ok) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + /* Ignore anything but regular files */ continue; values[0] = CStringGetTextDatum(de->d_name); values[1] = Int64GetDatum((int64) attrib.st_size); values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (dir_ok) + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + memset(nulls, 0, sizeof(nulls)); tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); @@ -613,14 +625,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, false, false); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, false, false); } /* @@ -638,7 +650,7 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, true, false); } /* @@ -667,5 +679,24 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true, false); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + bool missing_ok = false; + bool include_dot_dirs = false; + + if (!PG_ARGISNULL(1)) + missing_ok = PG_GETARG_BOOL(1); + if (!PG_ARGISNULL(2)) + /* XXX: Not implemented */ + include_dot_dirs = PG_GETARG_BOOL(2); + + return pg_ls_dir_files(fcinfo, dirname, missing_ok, true); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7fb574f9dc..0a1859f709 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10741,6 +10741,12 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool bool', + proallargtypes => '{text,bool,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,dir_ok,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', -- 2.17.0 --32u276st3Jlj2kUU Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0005-Add-tests-exercizing-pg_ls_tmpdir.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v9 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-08 22:15 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-08 22:15 UTC (permalink / raw) Need catversion bumped? --- src/backend/utils/adt/genfile.c | 51 ++++++++++++++++++++++++++------- src/include/catalog/pg_proc.dat | 6 ++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 897b11a77d..4699aea568 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -524,7 +524,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) /* Generic function to return a directory listing of files */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok, bool dir_ok) { FuncCallContext *funcctx; struct dirent *de; @@ -540,13 +540,17 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) fctx = palloc(sizeof(directory_fctx)); - tupdesc = CreateTemplateTupleDesc(3); + tupdesc = CreateTemplateTupleDesc(dir_ok ? 4:3); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "name", TEXTOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "size", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 3, "modification", TIMESTAMPTZOID, -1, 0); + if (dir_ok) + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "isdir", + BOOLOID, -1, 0); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); fctx->location = pstrdup(dir); @@ -575,8 +579,8 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; HeapTuple tuple; @@ -592,13 +596,21 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", path))); - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) + if (S_ISDIR(attrib.st_mode)) + { + if (!dir_ok) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + /* Ignore anything but regular files */ continue; values[0] = CStringGetTextDatum(de->d_name); values[1] = Int64GetDatum((int64) attrib.st_size); values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (dir_ok) + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + memset(nulls, 0, sizeof(nulls)); tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); @@ -613,14 +625,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, false, false); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, false, false); } /* @@ -638,7 +650,7 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, true, false); } /* @@ -667,5 +679,24 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true, false); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + bool missing_ok = false; + bool include_dot_dirs = false; + + if (!PG_ARGISNULL(1)) + missing_ok = PG_GETARG_BOOL(1); + if (!PG_ARGISNULL(2)) + /* XXX: Not implemented */ + include_dot_dirs = PG_GETARG_BOOL(2); + + return pg_ls_dir_files(fcinfo, dirname, missing_ok, true); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7fb574f9dc..0a1859f709 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10741,6 +10741,12 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool bool', + proallargtypes => '{text,bool,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,dir_ok,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', -- 2.17.0 --32u276st3Jlj2kUU Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0005-Add-tests-exercizing-pg_ls_tmpdir.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v15 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir XXX: Change to use lstat() to allow pg_ls_dir_recurse to avoid infinite recursion. That means: - links to dirs are NOT SHOWN (was: shown with isdir=false;) - NO: timestamps shown are those of the link; - changed pg_stat_file for consistency; Need catversion bumped? --- doc/src/sgml/func.sgml | 17 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 227 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 214 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 630f03d7dd..ed8e1977ba 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21372,6 +21372,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); List the contents of a directory. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_metadata(<parameter>dirname</parameter> <type>text</type> [, <parameter>missing_ok</parameter> <type>boolean</type>, <parameter>include_dot_dirs</parameter> <type>boolean</type>])</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21472,6 +21481,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <para> + <function>pg_ls_dir_metadata</function> lists the files in the specified + directory along with the file's metadata. + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 83d00c6cde..4511f9d658 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1436,6 +1436,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 219ac160f8..eaa7c548e0 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -413,6 +428,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +460,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +475,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +487,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +496,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +536,18 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } else { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,20 +566,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags&LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -606,13 +601,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -626,14 +642,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -651,7 +667,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -680,5 +697,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index a6a708cca9..e8846e1522 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10801,6 +10801,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '5033', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --8w3uRX/HFJGApMzv Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v15-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v37 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 205 ++++++++++++------- src/include/catalog/pg_proc.dat | 12 ++ src/test/regress/expected/misc_functions.out | 24 +++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 217 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d958c3e74ac..a80d794a43a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25900,6 +25900,27 @@ LOG: Grand total: 1651920 bytes in 201 blocks; 622360 free (88 chunks); 1029560 </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 30a048f6b09..b77b83017cb 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -704,6 +704,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index ab6f67f8747..c041c7630c8 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything beginning with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -516,6 +531,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -543,54 +563,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - DIR *dirdesc; - struct dirent *de; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - InitMaterializedSRF(fcinfo, MAT_SRF_USE_EXPECTED_DESC); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, - values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -603,23 +578,55 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; DIR *dirdesc; struct dirent *de; - InitMaterializedSRF(fcinfo, 0); + /* isdir depends on metadata */ + Assert(!(flags & LS_DIR_ISDIR) || (flags & LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags & LS_DIR_ISDIR) || !(flags & LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + /* missing_ok */ + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + /* include_dot_dirs */ + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } + + if (flags & LS_DIR_METADATA) + InitMaterializedSRF(fcinfo, 0); + else + InitMaterializedSRF(fcinfo, MAT_SRF_USE_EXPECTED_DESC); /* * Now walk the directory. Note that we must do this within a single SRF @@ -630,20 +637,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -658,13 +672,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -678,14 +714,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -703,7 +739,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -732,7 +769,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -741,7 +806,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -750,7 +815,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -775,5 +840,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 20f5aa56eab..89892647808 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11758,6 +11758,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 77d285ecc85..a656f9ad07e 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -539,6 +539,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index c52cf1cfcf9..8159c9f18f1 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -33,6 +33,14 @@ SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN' pg_tblspc/NNN (1 row) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index d299f3d8949..f808555382d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -181,6 +181,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 21db433f2a8..cf683c3bf3a 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -27,6 +27,11 @@ CREATE TABLESPACE regress_tblspace LOCATION ''; SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN') FROM pg_tablespace WHERE spcname = 'regress_tblspace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.25.1 --Pk/CTwBz1VvfPIDp Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v37-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v36 3/7] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 205 ++++++++++++------- src/include/catalog/pg_proc.dat | 12 ++ src/test/regress/expected/misc_functions.out | 24 +++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 217 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 933118203f4..ba30faa8441 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25940,6 +25940,27 @@ SELECT collation for ('foo' COLLATE "de_DE"); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 73da687d5dd..3fa6f9af0a8 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -700,6 +700,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 2bf52192567..c7322ed405a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything beginning with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -446,6 +461,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -473,54 +493,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - DIR *dirdesc; - struct dirent *de; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, - values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -533,23 +508,55 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; DIR *dirdesc; struct dirent *de; - SetSingleFuncCall(fcinfo, 0); + /* isdir depends on metadata */ + Assert(!(flags & LS_DIR_ISDIR) || (flags & LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags & LS_DIR_ISDIR) || !(flags & LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + /* missing_ok */ + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + /* include_dot_dirs */ + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } + + if (flags & LS_DIR_METADATA) + SetSingleFuncCall(fcinfo, 0); + else + SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); /* * Now walk the directory. Note that we must do this within a single SRF @@ -560,20 +567,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -588,13 +602,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -608,14 +644,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -633,7 +669,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -662,7 +699,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -671,7 +736,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -680,7 +745,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -705,5 +770,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 87aa571a331..0e5c977e69f 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11815,6 +11815,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 45544469af5..e54e38f54ad 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -475,6 +475,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index c52cf1cfcf9..8159c9f18f1 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -33,6 +33,14 @@ SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN' pg_tblspc/NNN (1 row) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 59d6e517503..fcb456f434d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -155,6 +155,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 21db433f2a8..cf683c3bf3a 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -27,6 +27,11 @@ CREATE TABLESPACE regress_tblspace LOCATION ''; SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN') FROM pg_tablespace WHERE spcname = 'regress_tblspace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.1 --4ybNbZnZ8tziJ7D6 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v36-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v34 03/15] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 203 ++++++++++++------- src/include/catalog/pg_proc.dat | 12 ++ src/test/regress/expected/misc_functions.out | 24 +++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 215 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 0be4743e3ed..c2ff924d021 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25976,6 +25976,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 81bac6f5812..b4d3609cce7 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -700,6 +700,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 1ed01620a1b..1878bac7eb3 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything beginning with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -446,6 +461,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -473,54 +493,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - DIR *dirdesc; - struct dirent *de; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, - values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -533,23 +508,53 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; DIR *dirdesc; struct dirent *de; - SetSingleFuncCall(fcinfo, 0); + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } + + if (flags & LS_DIR_METADATA) + SetSingleFuncCall(fcinfo, 0); + else + SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); /* * Now walk the directory. Note that we must do this within a single SRF @@ -560,20 +565,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -588,13 +600,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -608,14 +642,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -633,7 +667,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -662,7 +697,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -671,7 +734,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -680,7 +743,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -705,5 +768,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d8e8715ed1c..43ee0cdc454 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11677,6 +11677,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index b08e7c4a6d0..d18f401248a 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -452,6 +452,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index 2dfbcfdebe1..c6b9074ab02 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -24,6 +24,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION ''; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index ebb0352ac68..05794e1ae45 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -147,6 +147,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 896f05cea32..3aadac8b611 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -23,6 +23,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION ''; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.1 --smOfPzt+Qjm5bNGJ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v34-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v35 3/7] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 205 ++++++++++++------- src/include/catalog/pg_proc.dat | 12 ++ src/test/regress/expected/misc_functions.out | 24 +++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 217 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d01aeec9f88..d2a455a3e27 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25987,6 +25987,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 81bac6f5812..b4d3609cce7 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -700,6 +700,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 88f279d1b31..75b7bf99849 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything beginning with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -446,6 +461,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -473,54 +493,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - DIR *dirdesc; - struct dirent *de; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, - values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -533,23 +508,55 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; DIR *dirdesc; struct dirent *de; - SetSingleFuncCall(fcinfo, 0); + /* isdir depends on metadata */ + Assert(!(flags & LS_DIR_ISDIR) || (flags & LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags & LS_DIR_ISDIR) || !(flags & LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + /* missing_ok */ + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + /* include_dot_dirs */ + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } + + if (flags & LS_DIR_METADATA) + SetSingleFuncCall(fcinfo, 0); + else + SetSingleFuncCall(fcinfo, SRF_SINGLE_USE_EXPECTED); /* * Now walk the directory. Note that we must do this within a single SRF @@ -560,20 +567,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -588,13 +602,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -608,14 +644,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -633,7 +669,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -662,7 +699,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -671,7 +736,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -680,7 +745,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -705,5 +770,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 25304430f44..7307f44f371 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11769,6 +11769,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 45544469af5..e54e38f54ad 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -475,6 +475,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index c52cf1cfcf9..8159c9f18f1 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -33,6 +33,14 @@ SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN' pg_tblspc/NNN (1 row) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 59d6e517503..fcb456f434d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -155,6 +155,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 21db433f2a8..cf683c3bf3a 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -27,6 +27,11 @@ CREATE TABLESPACE regress_tblspace LOCATION ''; SELECT regexp_replace(pg_tablespace_location(oid), '(pg_tblspc)/(\d+)', '\1/NNN') FROM pg_tablespace WHERE spcname = 'regress_tblspace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.1 --olLTNZSltDMg5Vbm Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v35-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v32 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 239 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 225 insertions(+), 96 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index b5c1befe627..7948e6ec06c 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25945,6 +25945,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 3a4fa9091b1..5eb6cc8572c 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -702,6 +702,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 027ed864001..0728547ec2b 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +683,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +708,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,7 +738,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -728,7 +775,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -737,7 +784,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -762,5 +809,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4d992dc2241..c99e1633bbf 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11645,6 +11645,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 830de507e7c..b3a9d11b5c0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -267,6 +267,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index 864f4b6e208..21f7f19b67f 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -16,6 +16,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION :'testtablespace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 422a1369aeb..ca946d08bde 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -99,6 +99,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 92076db9a13..291ca029c14 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -16,6 +16,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION :'testtablespace'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.0 --Bne5rrxQd65beI7a Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v32-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v31 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 239 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 225 insertions(+), 96 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d36479d86d..e0099e77df 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25860,6 +25860,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index f6789025a5..a45bac89ff 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -698,6 +698,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 027ed86400..0728547ec2 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +683,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +708,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,7 +738,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -728,7 +775,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -737,7 +784,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -762,5 +809,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index e934361dc3..39fbaacbc9 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11645,6 +11645,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 830de507e7..b3a9d11b5c 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -267,6 +267,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index cb9774ecc8..e69fa17004 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index e7629d470e..c2f6d64c5c 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 422a1369ae..ca946d08bd 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -99,6 +99,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- -- 2.17.0 --qZVVwWJgpX9Jzs7f Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v31-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v31 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 239 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 225 insertions(+), 96 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d36479d86d..e0099e77df 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25860,6 +25860,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index f6789025a5..a45bac89ff 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -698,6 +698,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 027ed86400..0728547ec2 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +683,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +708,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,7 +738,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -728,7 +775,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -737,7 +784,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -762,5 +809,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index e934361dc3..39fbaacbc9 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11645,6 +11645,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 830de507e7..b3a9d11b5c 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -267,6 +267,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index cb9774ecc8..e69fa17004 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index e7629d470e..c2f6d64c5c 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 422a1369ae..ca946d08bd 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -99,6 +99,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- -- 2.17.0 --qZVVwWJgpX9Jzs7f Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v31-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v30 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 233 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 222 insertions(+), 93 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7a830f0684..dd9b1afde4 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25848,6 +25848,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index a416e94d37..649a78605a 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -698,6 +698,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index c436d9318b..8efb798b2f 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -451,6 +466,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -478,79 +498,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -563,17 +513,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -582,6 +534,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -596,8 +574,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -616,20 +606,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -644,13 +641,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -664,14 +682,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -689,7 +707,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -718,5 +737,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fde251fa4f..b405aa341c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11547,6 +11547,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index ea0fc48dbd..b4112ec298 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -238,6 +238,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index c133e73499..a32212be04 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 1bbe7e0323..5f7ce3d09f 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index eb6ac12ab4..c169e527d9 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -77,6 +77,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --ZwgA9U+XZDXt4+m+ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v30-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v28 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 233 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 222 insertions(+), 93 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index aa0dcde886..507a6d73f8 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25821,6 +25821,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 451db2ee0a..dd84b0f11a 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1632,6 +1632,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 322152ebd9..f59f853983 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +683,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +708,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,5 +738,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index f4957653ae..550fbf734a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11534,6 +11534,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index ea0fc48dbd..b4112ec298 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -238,6 +238,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index c133e73499..a32212be04 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 1bbe7e0323..5f7ce3d09f 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index eb6ac12ab4..c169e527d9 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -77,6 +77,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --vk/v8fjDPiDepTtA Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v28-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v25 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 231 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 221 insertions(+), 92 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index df29af6371..226923a134 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25750,6 +25750,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b140c210bc..93d4f1e4a6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1506,6 +1506,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..7351615f8a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,17 +511,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* * Generic function to return a directory listing of files. * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fc2202b843..8e88c549d9 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10976,6 +10976,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index edbfd9abc1..fd7c3c791f 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -225,6 +225,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index b32a3a4b74..64d9d70874 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -68,6 +68,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --SBikYMzjhZGK9d4p Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v25-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v24 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 231 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 221 insertions(+), 92 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 507bc1a668..5e267c4f8d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25757,6 +25757,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 2e4aa1c4b6..e7647787cf 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1503,6 +1503,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..7351615f8a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,17 +511,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* * Generic function to return a directory listing of files. * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index e7fbda9f81..f35a684dcf 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10975,6 +10975,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index edbfd9abc1..fd7c3c791f 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -225,6 +225,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index b32a3a4b74..64d9d70874 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -68,6 +68,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --mPTHnM80CEnHQ2WJ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v24-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v22 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7ef2ec9972..34c92903d8 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25747,6 +25747,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index c6dd084fbc..169602c1b6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1509,6 +1509,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..054b8d4b1a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,7 +511,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -571,7 +523,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 24ec2cfed6..02b1436d83 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10964,6 +10964,18 @@ proallargtypes => '{regclass,text,text,int8}', proargmodes => '{i,i,o,o}', proargnames => '{relation,fork,path,failed_block_num}', prosrc => 'pg_relation_check_pages' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..20bfe71822 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..68e2bc6586 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --d6Gm4EdcadzBjdND Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v22-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v20 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 959f6a1c2f..eead88419f 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25741,6 +25741,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b6d35c2d11..360b6bda26 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1478,6 +1478,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 46d0977c2e..3324ffe16a 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,7 +511,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -571,7 +523,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 95604e988a..186ced4a35 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10919,6 +10919,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Z1Z8UV8BNhgCynIS Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v20-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v21 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 959f6a1c2f..eead88419f 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25741,6 +25741,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 8625cbeab6..b9b2a6aa20 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1478,6 +1478,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 9f4927220b..24d858d1bc 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -449,6 +464,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -476,79 +496,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -561,7 +511,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -571,7 +523,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -580,6 +532,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -594,8 +572,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -614,20 +604,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -642,13 +639,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -662,14 +680,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -687,7 +705,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -716,5 +735,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 95604e988a..83927b510a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10919,6 +10919,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..20bfe71822 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..68e2bc6586 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Tcb1KvpfnM4LxW2s Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v21-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v19 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 229 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 220 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index b7c450ea29..2931f9d7d5 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25711,6 +25711,27 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type>] </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 5314e9348f..f5217a0d93 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1481,6 +1481,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 219ac160f8..4824a55480 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -413,6 +428,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +460,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +475,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +487,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +496,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +536,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,20 +568,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -606,13 +603,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -626,14 +644,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -651,7 +669,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -680,5 +699,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 61f2c2f5b4..5b0147ee2d 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10902,6 +10902,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '9979', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '9980', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --S0GG+JvAI2G0KxBG Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v19-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v15 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir XXX: Change to use lstat() to allow pg_ls_dir_recurse to avoid infinite recursion. That means: - links to dirs are NOT SHOWN (was: shown with isdir=false;) - NO: timestamps shown are those of the link; - changed pg_stat_file for consistency; Need catversion bumped? --- doc/src/sgml/func.sgml | 17 ++ src/backend/catalog/system_views.sql | 1 + src/backend/utils/adt/genfile.c | 227 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 214 insertions(+), 91 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 630f03d7dd..ed8e1977ba 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21372,6 +21372,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); List the contents of a directory. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_metadata(<parameter>dirname</parameter> <type>text</type> [, <parameter>missing_ok</parameter> <type>boolean</type>, <parameter>include_dot_dirs</parameter> <type>boolean</type>])</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21472,6 +21481,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <para> + <function>pg_ls_dir_metadata</function> lists the files in the specified + directory along with the file's metadata. + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 83d00c6cde..4511f9d658 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1436,6 +1436,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 219ac160f8..eaa7c548e0 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata. + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -413,6 +428,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -440,79 +460,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -525,7 +475,9 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -535,7 +487,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -544,6 +496,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -558,8 +536,18 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } else { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -578,20 +566,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags&LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -606,13 +601,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -626,14 +642,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -651,7 +667,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -680,5 +697,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Function to return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index a6a708cca9..e8846e1522 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10801,6 +10801,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '5032', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '5033', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,name,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 2e87c548eb..7930909f02 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -219,6 +219,30 @@ select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false ERROR: could not open directory "does not exist": No such file or directory +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + name | isdir +------+------- + . | t +(1 row) + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + name | isdir +------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index a5f61a35dc..0b9cfe615e 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 162b591b31..a42714bf40 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f6857ad177..372345720d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -65,6 +65,17 @@ select * from (select pg_ls_dir('.', false, false) as name) as ls where ls.name= select pg_ls_dir('does not exist', true, false); -- ok with missingok=true select pg_ls_dir('does not exist'); -- fails with missingok=false +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select name, isdir from pg_ls_dir_metadata('.') where name='.'; + +select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --8w3uRX/HFJGApMzv Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v15-0005-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v33 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 240 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/expected/tablespace.out | 8 + src/test/regress/sql/misc_functions.sql | 11 + src/test/regress/sql/tablespace.sql | 5 + 8 files changed, 226 insertions(+), 96 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d32b414e04f..14b8fba3123 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25958,6 +25958,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index fd1421788e6..2ffc2f9d1a6 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -702,6 +702,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM PUBLIC; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 542bbacaa24..3921723fda6 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -37,6 +37,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -452,6 +467,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -479,79 +499,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -564,17 +514,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -583,6 +535,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -597,8 +575,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -617,20 +607,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -645,13 +642,35 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); + else +#endif + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -665,14 +684,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -690,7 +709,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -719,7 +739,35 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } /* @@ -728,7 +776,7 @@ pg_ls_archive_statusdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", false); + return pg_ls_dir_files(fcinfo, "pg_logical/snapshots", LS_DIR_HISTORIC); } /* @@ -737,7 +785,7 @@ pg_ls_logicalsnapdir(PG_FUNCTION_ARGS) Datum pg_ls_logicalmapdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, "pg_logical/mappings", false); + return pg_ls_dir_files(fcinfo, "pg_logical/mappings", LS_DIR_HISTORIC); } /* @@ -762,5 +810,5 @@ pg_ls_replslotdir(PG_FUNCTION_ARGS) slotname))); snprintf(path, sizeof(path), "pg_replslot/%s", slotname); - return pg_ls_dir_files(fcinfo, path, false); + return pg_ls_dir_files(fcinfo, path, LS_DIR_HISTORIC); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0859dc81cac..b58987b713c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11661,6 +11661,18 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index b7264af3ecb..a627839bc28 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -309,6 +309,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out index 2dfbcfdebe1..c6b9074ab02 100644 --- a/src/test/regress/expected/tablespace.out +++ b/src/test/regress/expected/tablespace.out @@ -24,6 +24,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION ''; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 169f66416c3..bfa8e309e9a 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -109,6 +109,17 @@ select * from pg_ls_tmpdir() limit 0; select * from pg_ls_waldir() limit 0; select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test replication slot directory functions -- diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql index 896f05cea32..3aadac8b611 100644 --- a/src/test/regress/sql/tablespace.sql +++ b/src/test/regress/sql/tablespace.sql @@ -23,6 +23,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION ''; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail -- 2.17.1 --9CzcV6dAFIr7O1Ie Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v33-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v30 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. @ 2020-03-10 03:40 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 22+ messages in thread From: Justin Pryzby @ 2020-03-10 03:40 UTC (permalink / raw) Generalize pg_ls_dir_files and retire pg_ls_dir Need catversion bumped? --- doc/src/sgml/func.sgml | 21 ++ src/backend/catalog/system_functions.sql | 1 + src/backend/utils/adt/genfile.c | 233 +++++++++++-------- src/include/catalog/pg_proc.dat | 12 + src/test/regress/expected/misc_functions.out | 24 ++ src/test/regress/input/tablespace.source | 5 + src/test/regress/output/tablespace.source | 8 + src/test/regress/sql/misc_functions.sql | 11 + 8 files changed, 222 insertions(+), 93 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7a830f0684..dd9b1afde4 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25848,6 +25848,27 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_metadata</primary> + </indexterm> + <function>pg_ls_dir_metadata</function> ( <parameter>dirname</parameter> <type>text</type> + <optional>, <parameter>missing_ok</parameter> <type>boolean</type>, + <parameter>include_dot_dirs</parameter> <type>boolean</type> </optional> ) + <returnvalue>setof record</returnvalue> + ( <parameter>filename</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type> ) + </para> + <para> + For each file in the specified directory, list the file and its + metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index a416e94d37..649a78605a 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -698,6 +698,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index c436d9318b..8efb798b2f 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,6 +36,21 @@ #include "utils/syscache.h" #include "utils/timestamp.h" +static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags); + +#define LS_DIR_ISDIR (1<<0) /* Show column: isdir */ +#define LS_DIR_METADATA (1<<1) /* Show columns: mtime, size */ +#define LS_DIR_MISSING_OK (1<<2) /* Ignore ENOENT if the toplevel dir is missing */ +#define LS_DIR_SKIP_DOT_DIRS (1<<3) /* Do not show . or .. */ +#define LS_DIR_SKIP_HIDDEN (1<<4) /* Do not show anything begining with . */ +#define LS_DIR_SKIP_DIRS (1<<5) /* Do not show directories */ +#define LS_DIR_SKIP_SPECIAL (1<<6) /* Do not show special file types */ + +/* + * Shortcut for the historic behavior of the pg_ls_* functions (not including + * pg_ls_dir, which skips different files and doesn't show metadata). + */ +#define LS_DIR_HISTORIC (LS_DIR_SKIP_DIRS | LS_DIR_SKIP_HIDDEN | LS_DIR_SKIP_SPECIAL | LS_DIR_METADATA) /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -451,6 +466,11 @@ pg_stat_file(PG_FUNCTION_ARGS) values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime)); #endif values[5] = BoolGetDatum(S_ISDIR(fst.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(filename)) + values[5] = BoolGetDatum(false); +#endif tuple = heap_form_tuple(tupdesc, values, isnull); @@ -478,79 +498,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - char *location; - bool missing_ok = false; - bool include_dot_dirs = false; - bool randomAccess; - TupleDesc tupdesc; - Tuplestorestate *tupstore; - DIR *dirdesc; - struct dirent *de; - MemoryContext oldcontext; - - location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); - - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } - - /* check to see if caller supports us returning a tuplestore */ - if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("set-valued function called in context that cannot accept a set"))); - if (!(rsinfo->allowedModes & SFRM_Materialize)) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not allowed in this context"))); - - /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ - oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - - tupdesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - - randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; - tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - rsinfo->returnMode = SFRM_Materialize; - rsinfo->setResult = tupstore; - rsinfo->setDesc = tupdesc; - - MemoryContextSwitchTo(oldcontext); - - dirdesc = AllocateDir(location); - if (!dirdesc) - { - /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) - return (Datum) 0; - /* Otherwise, we can let ReadDir() throw the error */ - } - - while ((de = ReadDir(dirdesc, location)) != NULL) - { - Datum values[1]; - bool nulls[1]; - - if (!include_dot_dirs && - (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0)) - continue; - - values[0] = CStringGetTextDatum(de->d_name); - nulls[0] = false; - - tuplestore_putvalues(tupstore, tupdesc, values, nulls); - } - - FreeDir(dirdesc); - return (Datum) 0; + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* @@ -563,17 +513,19 @@ pg_ls_dir(PG_FUNCTION_ARGS) Datum pg_ls_dir_1arg(PG_FUNCTION_ARGS) { - return pg_ls_dir(fcinfo); + text *filename_t = PG_GETARG_TEXT_PP(0); + char *filename = convert_and_check_filename(filename_t); + return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS); } /* - * Generic function to return a directory listing of files. + * Generic function to return a directory listing of files (and optionally dirs). * - * If the directory isn't there, silently return an empty set if missing_ok. + * If the directory isn't there, silently return an empty set if MISSING_OK. * Other unreadable-directory cases throw an error. */ static Datum -pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) +pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool randomAccess; @@ -582,6 +534,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) DIR *dirdesc; struct dirent *de; MemoryContext oldcontext; + TypeFuncClass tuptype ; + + /* isdir depends on metadata */ + Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA)); + /* Unreasonable to show isdir and skip dirs */ + Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) + { + if (!PG_ARGISNULL(1)) + { + if (PG_GETARG_BOOL(1)) + flags |= LS_DIR_MISSING_OK; + else + flags &= ~LS_DIR_MISSING_OK; + } + + if (!PG_ARGISNULL(2)) + { + if (PG_GETARG_BOOL(2)) + flags &= ~LS_DIR_SKIP_DOT_DIRS; + else + flags |= LS_DIR_SKIP_DOT_DIRS; + } + } /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) @@ -596,8 +574,20 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) - elog(ERROR, "return type must be a row type"); + tuptype = get_call_result_type(fcinfo, NULL, &tupdesc); + if (flags & LS_DIR_METADATA) + { + if (tuptype != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + } + else + { + /* pg_ls_dir returns a simple scalar */ + if (tuptype != TYPEFUNC_SCALAR) + elog(ERROR, "return type must be a scalar type"); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0); + } randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); @@ -616,20 +606,27 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!dirdesc) { /* Return empty tuplestore if appropriate */ - if (missing_ok && errno == ENOENT) + if (flags & LS_DIR_MISSING_OK && errno == ENOENT) return (Datum) 0; /* Otherwise, we can let ReadDir() throw the error */ } while ((de = ReadDir(dirdesc, dir)) != NULL) { - Datum values[3]; - bool nulls[3]; + Datum values[4]; + bool nulls[4]; char path[MAXPGPATH * 2]; struct stat attrib; - /* Skip hidden files */ - if (de->d_name[0] == '.') + /* Skip dot dirs? */ + if (flags & LS_DIR_SKIP_DOT_DIRS && + (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0)) + continue; + + /* Skip hidden files? */ + if (flags & LS_DIR_SKIP_HIDDEN && + de->d_name[0] == '.') continue; /* Get the file info */ @@ -644,13 +641,34 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) errmsg("could not stat file \"%s\": %m", path))); } - /* Ignore anything but regular files */ - if (!S_ISREG(attrib.st_mode)) - continue; + /* Skip dirs or special files? */ + if (S_ISDIR(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_DIRS) + continue; + } + else if (!S_ISREG(attrib.st_mode)) + { + if (flags & LS_DIR_SKIP_SPECIAL) + continue; + } values[0] = CStringGetTextDatum(de->d_name); - values[1] = Int64GetDatum((int64) attrib.st_size); - values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_METADATA) + { + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + if (flags & LS_DIR_ISDIR) + { + values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode)); +#ifdef WIN32 + /* Links should have isdir=false */ + if (pgwin32_is_junction(path)) + values[3] = BoolGetDatum(false); +#endif + } + } + memset(nulls, 0, sizeof(nulls)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -664,14 +682,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) Datum pg_ls_logdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, Log_directory, false); + return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC); } /* Function to return the list of files in the WAL directory */ Datum pg_ls_waldir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR, false); + return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC); } /* @@ -689,7 +707,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc) tblspc))); TempTablespacePath(path, tblspc); - return pg_ls_dir_files(fcinfo, path, true); + return pg_ls_dir_files(fcinfo, path, + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); } /* @@ -718,5 +737,33 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS) Datum pg_ls_archive_statusdir(PG_FUNCTION_ARGS) { - return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true); + return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", + LS_DIR_HISTORIC | LS_DIR_MISSING_OK); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + */ +Datum +pg_ls_dir_metadata(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); +} + +/* + * Return the list of files and metadata in an arbitrary directory. + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments. + */ +Datum +pg_ls_dir_metadata_1arg(PG_FUNCTION_ARGS) +{ + char *dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + return pg_ls_dir_files(fcinfo, dirname, + LS_DIR_METADATA | LS_DIR_SKIP_SPECIAL | LS_DIR_ISDIR); } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fde251fa4f..b405aa341c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11547,6 +11547,18 @@ proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}', proargnames => '{tablespace,name,size,modification}', prosrc => 'pg_ls_tmpdir_1arg' }, +{ oid => '8450', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool', + proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', + proargnames => '{dirname,missing_ok,include_dot_dirs,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8451', descr => 'list directory with metadata', + proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', + proargnames => '{dirname,filename,size,modification,isdir}', + prosrc => 'pg_ls_dir_metadata_1arg' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index ea0fc48dbd..b4112ec298 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -238,6 +238,30 @@ select * from pg_stat_file('.') limit 0; ------+--------+--------------+--------+----------+------- (0 rows) +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + filename | isdir +----------+------- + . | t +(1 row) + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + filename | isdir +----------+------- +(0 rows) + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + filename | size | modification | isdir +----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source index c133e73499..a32212be04 100644 --- a/src/test/regress/input/tablespace.source +++ b/src/test/regress/input/tablespace.source @@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source index 1bbe7e0323..5f7ce3d09f 100644 --- a/src/test/regress/output/tablespace.source +++ b/src/test/regress/output/tablespace.source @@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith'; DROP TABLESPACE regress_tblspacewith; -- create a tablespace we can use CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@'; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir() +SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1) AS b , pg_ls_tmpdir(oid) AS c WHERE c.name='Does not exist'; + name | size | modification +------+------+-------------- +(0 rows) + -- try setting and resetting some properties for the new tablespace ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1); ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true); -- fail diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index eb6ac12ab4..c169e527d9 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -77,6 +77,17 @@ select pg_ls_dir('does not exist'); -- fails with missingok=false -- Check that expected columns are present select * from pg_stat_file('.') limit 0; +-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet +-- The name='' condition is never true, so the function runs to completion but returns zero rows. +select * from pg_ls_tmpdir() where name='Does not exist'; + +select filename, isdir from pg_ls_dir_metadata('.') where filename='.'; + +select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename='.'; -- include_dot_dirs=false + +-- Check that expected columns are present +select * from pg_ls_dir_metadata('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --ZwgA9U+XZDXt4+m+ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v30-0004-pg_ls_tmpdir-to-show-directories-and-isdir-argum.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
* [PATCH v1 1/5] Allow to use multiple shared memory mappings @ 2024-10-09 13:41 Dmitrii Dolgov <9erthalion6@gmail.com> 0 siblings, 0 replies; 22+ messages in thread From: Dmitrii Dolgov @ 2024-10-09 13:41 UTC (permalink / raw) Currently all the work with shared memory is done via a single anonymous memory mapping, which limits ways how the shared memory could be organized. Introduce possibility to allocate multiple shared memory mappings, where a single mapping is associated with a specified shared memory slot. There is only fixed amount of available slots, currently only one main shared memory slot is allocated. A new shared memory API is introduces, extended with a slot as a new parameter. As a path of least resistance, the original API is kept in place, utilizing the main shared memory slot. --- src/backend/port/posix_sema.c | 4 +- src/backend/port/sysv_sema.c | 4 +- src/backend/port/sysv_shmem.c | 138 +++++++++++++++++++--------- src/backend/port/win32_sema.c | 2 +- src/backend/storage/ipc/ipc.c | 2 +- src/backend/storage/ipc/ipci.c | 61 ++++++------ src/backend/storage/ipc/shmem.c | 133 ++++++++++++++++++--------- src/backend/storage/lmgr/lwlock.c | 5 +- src/include/storage/buf_internals.h | 1 + src/include/storage/ipc.h | 2 +- src/include/storage/pg_sema.h | 2 +- src/include/storage/pg_shmem.h | 18 ++++ src/include/storage/shmem.h | 10 ++ 13 files changed, 258 insertions(+), 124 deletions(-) diff --git a/src/backend/port/posix_sema.c b/src/backend/port/posix_sema.c index 64186ec0a7..b97723d2ed 100644 --- a/src/backend/port/posix_sema.c +++ b/src/backend/port/posix_sema.c @@ -193,7 +193,7 @@ PGSemaphoreShmemSize(int maxSemas) * we don't have to expose the counters to other processes.) */ void -PGReserveSemaphores(int maxSemas) +PGReserveSemaphores(int maxSemas, int shmem_slot) { struct stat statbuf; @@ -220,7 +220,7 @@ PGReserveSemaphores(int maxSemas) * ShmemAlloc() won't be ready yet. */ sharedSemas = (PGSemaphore) - ShmemAllocUnlocked(PGSemaphoreShmemSize(maxSemas)); + ShmemAllocUnlockedInSlot(PGSemaphoreShmemSize(maxSemas), shmem_slot); #endif numSems = 0; diff --git a/src/backend/port/sysv_sema.c b/src/backend/port/sysv_sema.c index 5b88a92bc9..8ef95b12c9 100644 --- a/src/backend/port/sysv_sema.c +++ b/src/backend/port/sysv_sema.c @@ -307,7 +307,7 @@ PGSemaphoreShmemSize(int maxSemas) * have clobbered.) */ void -PGReserveSemaphores(int maxSemas) +PGReserveSemaphores(int maxSemas, int shmem_slot) { struct stat statbuf; @@ -328,7 +328,7 @@ PGReserveSemaphores(int maxSemas) * ShmemAlloc() won't be ready yet. */ sharedSemas = (PGSemaphore) - ShmemAllocUnlocked(PGSemaphoreShmemSize(maxSemas)); + ShmemAllocUnlockedInSlot(PGSemaphoreShmemSize(maxSemas), shmem_slot); numSharedSemas = 0; maxSharedSemas = maxSemas; diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c index 362a37d3b3..065a5b63ac 100644 --- a/src/backend/port/sysv_shmem.c +++ b/src/backend/port/sysv_shmem.c @@ -94,8 +94,19 @@ typedef enum unsigned long UsedShmemSegID = 0; void *UsedShmemSegAddr = NULL; -static Size AnonymousShmemSize; -static void *AnonymousShmem = NULL; +typedef struct AnonymousMapping +{ + int shmem_slot; + Size shmem_size; /* Size of the mapping */ + void *shmem; /* Pointer to the start of the mapped memory */ + void *seg_addr; /* SysV shared memory for the header */ + unsigned long seg_id; /* IPC key */ +} AnonymousMapping; + +static AnonymousMapping Mappings[ANON_MAPPINGS]; + +/* Keeps track of used mapping slots */ +static int next_free_slot = 0; static void *InternalIpcMemoryCreate(IpcMemoryKey memKey, Size size); static void IpcMemoryDetach(int status, Datum shmaddr); @@ -104,6 +115,28 @@ static IpcMemoryState PGSharedMemoryAttach(IpcMemoryId shmId, void *attachAt, PGShmemHeader **addr); +static const char* +MappingName(int shmem_slot) +{ + switch (shmem_slot) + { + case MAIN_SHMEM_SLOT: + return "main"; + default: + return "unknown"; + } +} + +static void +DebugMappings() +{ + for(int i = 0; i < next_free_slot; i++) + { + AnonymousMapping m = Mappings[i]; + elog(DEBUG1, "Mapping[%s]: addr %p, size %zu", + MappingName(i), m.shmem, m.shmem_size); + } +} /* * InternalIpcMemoryCreate(memKey, size) @@ -591,14 +624,13 @@ check_huge_page_size(int *newval, void **extra, GucSource source) /* * Creates an anonymous mmap()ed shared memory segment. * - * Pass the requested size in *size. This function will modify *size to the - * actual size of the allocation, if it ends up allocating a segment that is - * larger than requested. + * This function will modify mapping size to the actual size of the allocation, + * if it ends up allocating a segment that is larger than requested. */ -static void * -CreateAnonymousSegment(Size *size) +static void +CreateAnonymousSegment(AnonymousMapping *mapping) { - Size allocsize = *size; + Size allocsize = mapping->shmem_size; void *ptr = MAP_FAILED; int mmap_errno = 0; @@ -623,8 +655,11 @@ CreateAnonymousSegment(Size *size) PG_MMAP_FLAGS | mmap_flags, -1, 0); mmap_errno = errno; if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED) - elog(DEBUG1, "mmap(%zu) with MAP_HUGETLB failed, huge pages disabled: %m", - allocsize); + { + DebugMappings(); + elog(DEBUG1, "slot[%s]: mmap(%zu) with MAP_HUGETLB failed, huge pages disabled: %m", + MappingName(mapping->shmem_slot), allocsize); + } } #endif @@ -642,7 +677,7 @@ CreateAnonymousSegment(Size *size) * Use the original size, not the rounded-up value, when falling back * to non-huge pages. */ - allocsize = *size; + allocsize = mapping->shmem_size; ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE, PG_MMAP_FLAGS, -1, 0); mmap_errno = errno; @@ -651,8 +686,10 @@ CreateAnonymousSegment(Size *size) if (ptr == MAP_FAILED) { errno = mmap_errno; + DebugMappings(); ereport(FATAL, - (errmsg("could not map anonymous shared memory: %m"), + (errmsg("slot[%s]: could not map anonymous shared memory: %m", + MappingName(mapping->shmem_slot)), (mmap_errno == ENOMEM) ? errhint("This error usually means that PostgreSQL's request " "for a shared memory segment exceeded available memory, " @@ -663,8 +700,8 @@ CreateAnonymousSegment(Size *size) allocsize) : 0)); } - *size = allocsize; - return ptr; + mapping->shmem = ptr; + mapping->shmem_size = allocsize; } /* @@ -674,13 +711,18 @@ CreateAnonymousSegment(Size *size) static void AnonymousShmemDetach(int status, Datum arg) { - /* Release anonymous shared memory block, if any. */ - if (AnonymousShmem != NULL) + for(int i = 0; i < next_free_slot; i++) { - if (munmap(AnonymousShmem, AnonymousShmemSize) < 0) - elog(LOG, "munmap(%p, %zu) failed: %m", - AnonymousShmem, AnonymousShmemSize); - AnonymousShmem = NULL; + AnonymousMapping m = Mappings[i]; + + /* Release anonymous shared memory block, if any. */ + if (m.shmem != NULL) + { + if (munmap(m.shmem, m.shmem_size) < 0) + elog(LOG, "munmap(%p, %zu) failed: %m", + m.shmem, m.shmem_size); + m.shmem = NULL; + } } } @@ -705,6 +747,7 @@ PGSharedMemoryCreate(Size size, PGShmemHeader *hdr; struct stat statbuf; Size sysvsize; + AnonymousMapping *mapping = &Mappings[next_free_slot]; /* * We use the data directory's ID info (inode and device numbers) to @@ -733,11 +776,15 @@ PGSharedMemoryCreate(Size size, /* Room for a header? */ Assert(size > MAXALIGN(sizeof(PGShmemHeader))); + mapping->shmem_size = size; + mapping->shmem_slot = next_free_slot; if (shared_memory_type == SHMEM_TYPE_MMAP) { - AnonymousShmem = CreateAnonymousSegment(&size); - AnonymousShmemSize = size; + /* On success, mapping data will be modified. */ + CreateAnonymousSegment(mapping); + + next_free_slot++; /* Register on-exit routine to unmap the anonymous segment */ on_shmem_exit(AnonymousShmemDetach, (Datum) 0); @@ -760,7 +807,7 @@ PGSharedMemoryCreate(Size size, * loop simultaneously. (CreateDataDirLockFile() does not entirely ensure * that, but prefer fixing it over coping here.) */ - NextShmemSegID = statbuf.st_ino; + NextShmemSegID = statbuf.st_ino + next_free_slot; for (;;) { @@ -852,13 +899,13 @@ PGSharedMemoryCreate(Size size, /* * Initialize space allocation status for segment. */ - hdr->totalsize = size; + hdr->totalsize = mapping->shmem_size; hdr->freeoffset = MAXALIGN(sizeof(PGShmemHeader)); *shim = hdr; /* Save info for possible future use */ - UsedShmemSegAddr = memAddress; - UsedShmemSegID = (unsigned long) NextShmemSegID; + mapping->seg_addr = memAddress; + mapping->seg_id = (unsigned long) NextShmemSegID; /* * If AnonymousShmem is NULL here, then we're not using anonymous shared @@ -866,10 +913,10 @@ PGSharedMemoryCreate(Size size, * block. Otherwise, the System V shared memory block is only a shim, and * we must return a pointer to the real block. */ - if (AnonymousShmem == NULL) + if (mapping->shmem == NULL) return hdr; - memcpy(AnonymousShmem, hdr, sizeof(PGShmemHeader)); - return (PGShmemHeader *) AnonymousShmem; + memcpy(mapping->shmem, hdr, sizeof(PGShmemHeader)); + return (PGShmemHeader *) mapping->shmem; } #ifdef EXEC_BACKEND @@ -969,23 +1016,28 @@ PGSharedMemoryNoReAttach(void) void PGSharedMemoryDetach(void) { - if (UsedShmemSegAddr != NULL) + for(int i = 0; i < next_free_slot; i++) { - if ((shmdt(UsedShmemSegAddr) < 0) + AnonymousMapping m = Mappings[i]; + + if (m.seg_addr != NULL) + { + if ((shmdt(m.seg_addr) < 0) #if defined(EXEC_BACKEND) && defined(__CYGWIN__) - /* Work-around for cygipc exec bug */ - && shmdt(NULL) < 0 + /* Work-around for cygipc exec bug */ + && shmdt(NULL) < 0 #endif - ) - elog(LOG, "shmdt(%p) failed: %m", UsedShmemSegAddr); - UsedShmemSegAddr = NULL; - } + ) + elog(LOG, "shmdt(%p) failed: %m", m.seg_addr); + m.seg_addr = NULL; + } - if (AnonymousShmem != NULL) - { - if (munmap(AnonymousShmem, AnonymousShmemSize) < 0) - elog(LOG, "munmap(%p, %zu) failed: %m", - AnonymousShmem, AnonymousShmemSize); - AnonymousShmem = NULL; + if (m.shmem != NULL) + { + if (munmap(m.shmem, m.shmem_size) < 0) + elog(LOG, "munmap(%p, %zu) failed: %m", + m.shmem, m.shmem_size); + m.shmem = NULL; + } } } diff --git a/src/backend/port/win32_sema.c b/src/backend/port/win32_sema.c index f2b54bdfda..d62084cc0d 100644 --- a/src/backend/port/win32_sema.c +++ b/src/backend/port/win32_sema.c @@ -44,7 +44,7 @@ PGSemaphoreShmemSize(int maxSemas) * process exits. */ void -PGReserveSemaphores(int maxSemas) +PGReserveSemaphores(int maxSemas, int shmem_slot) { mySemSet = (HANDLE *) malloc(maxSemas * sizeof(HANDLE)); if (mySemSet == NULL) diff --git a/src/backend/storage/ipc/ipc.c b/src/backend/storage/ipc/ipc.c index b06e4b8452..2aabd4a77f 100644 --- a/src/backend/storage/ipc/ipc.c +++ b/src/backend/storage/ipc/ipc.c @@ -68,7 +68,7 @@ static void proc_exit_prepare(int code); * ---------------------------------------------------------------- */ -#define MAX_ON_EXITS 20 +#define MAX_ON_EXITS 40 struct ONEXIT { diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 35fa2e1dda..8224015b53 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -88,7 +88,7 @@ RequestAddinShmemSpace(Size size) * required. */ Size -CalculateShmemSize(int *num_semaphores) +CalculateShmemSize(int *num_semaphores, int shmem_slot) { Size size; int numSemas; @@ -202,33 +202,36 @@ CreateSharedMemoryAndSemaphores(void) Assert(!IsUnderPostmaster); - /* Compute the size of the shared-memory block */ - size = CalculateShmemSize(&numSemas); - elog(DEBUG3, "invoking IpcMemoryCreate(size=%zu)", size); - - /* - * Create the shmem segment - */ - seghdr = PGSharedMemoryCreate(size, &shim); - - /* - * Make sure that huge pages are never reported as "unknown" while the - * server is running. - */ - Assert(strcmp("unknown", - GetConfigOption("huge_pages_status", false, false)) != 0); - - InitShmemAccess(seghdr); - - /* - * Create semaphores - */ - PGReserveSemaphores(numSemas); - - /* - * Set up shared memory allocation mechanism - */ - InitShmemAllocation(); + for(int slot = 0; slot < ANON_MAPPINGS; slot++) + { + /* Compute the size of the shared-memory block */ + size = CalculateShmemSize(&numSemas, slot); + elog(DEBUG3, "invoking IpcMemoryCreate(size=%zu)", size); + + /* + * Create the shmem segment + */ + seghdr = PGSharedMemoryCreate(size, &shim); + + /* + * Make sure that huge pages are never reported as "unknown" while the + * server is running. + */ + Assert(strcmp("unknown", + GetConfigOption("huge_pages_status", false, false)) != 0); + + InitShmemAccessInSlot(seghdr, slot); + + /* + * Create semaphores + */ + PGReserveSemaphores(numSemas, slot); + + /* + * Set up shared memory allocation mechanism + */ + InitShmemAllocationInSlot(slot); + } /* Initialize subsystems */ CreateOrAttachShmemStructs(); @@ -359,7 +362,7 @@ InitializeShmemGUCs(void) /* * Calculate the shared memory size and round up to the nearest megabyte. */ - size_b = CalculateShmemSize(&num_semas); + size_b = CalculateShmemSize(&num_semas, MAIN_SHMEM_SLOT); size_mb = add_size(size_b, (1024 * 1024) - 1) / (1024 * 1024); sprintf(buf, "%zu", size_mb); SetConfigOption("shared_memory_size", buf, diff --git a/src/backend/storage/ipc/shmem.c b/src/backend/storage/ipc/shmem.c index 6d5f083986..c670b9cf43 100644 --- a/src/backend/storage/ipc/shmem.c +++ b/src/backend/storage/ipc/shmem.c @@ -75,17 +75,12 @@ #include "utils/builtins.h" static void *ShmemAllocRaw(Size size, Size *allocated_size); +static void *ShmemAllocRawInSlot(Size size, Size *allocated_size, + int shmem_slot); /* shared memory global variables */ -static PGShmemHeader *ShmemSegHdr; /* shared mem segment header */ - -static void *ShmemBase; /* start address of shared memory */ - -static void *ShmemEnd; /* end+1 address of shared memory */ - -slock_t *ShmemLock; /* spinlock for shared memory and LWLock - * allocation */ +ShmemSegment Segments[ANON_MAPPINGS]; static HTAB *ShmemIndex = NULL; /* primary index hashtable for shmem */ @@ -99,11 +94,17 @@ static HTAB *ShmemIndex = NULL; /* primary index hashtable for shmem */ void InitShmemAccess(void *seghdr) { - PGShmemHeader *shmhdr = (PGShmemHeader *) seghdr; + InitShmemAccessInSlot(seghdr, MAIN_SHMEM_SLOT); +} - ShmemSegHdr = shmhdr; - ShmemBase = (void *) shmhdr; - ShmemEnd = (char *) ShmemBase + shmhdr->totalsize; +void +InitShmemAccessInSlot(void *seghdr, int shmem_slot) +{ + PGShmemHeader *shmhdr = (PGShmemHeader *) seghdr; + ShmemSegment *seg = &Segments[shmem_slot]; + seg->ShmemSegHdr = shmhdr; + seg->ShmemBase = (void *) shmhdr; + seg->ShmemEnd = (char *) seg->ShmemBase + shmhdr->totalsize; } /* @@ -114,7 +115,13 @@ InitShmemAccess(void *seghdr) void InitShmemAllocation(void) { - PGShmemHeader *shmhdr = ShmemSegHdr; + InitShmemAllocationInSlot(MAIN_SHMEM_SLOT); +} + +void +InitShmemAllocationInSlot(int shmem_slot) +{ + PGShmemHeader *shmhdr = Segments[shmem_slot].ShmemSegHdr; char *aligned; Assert(shmhdr != NULL); @@ -123,9 +130,9 @@ InitShmemAllocation(void) * Initialize the spinlock used by ShmemAlloc. We must use * ShmemAllocUnlocked, since obviously ShmemAlloc can't be called yet. */ - ShmemLock = (slock_t *) ShmemAllocUnlocked(sizeof(slock_t)); + Segments[shmem_slot].ShmemLock = (slock_t *) ShmemAllocUnlockedInSlot(sizeof(slock_t), shmem_slot); - SpinLockInit(ShmemLock); + SpinLockInit(Segments[shmem_slot].ShmemLock); /* * Allocations after this point should go through ShmemAlloc, which @@ -150,11 +157,17 @@ InitShmemAllocation(void) */ void * ShmemAlloc(Size size) +{ + return ShmemAllocInSlot(size, MAIN_SHMEM_SLOT); +} + +void * +ShmemAllocInSlot(Size size, int shmem_slot) { void *newSpace; Size allocated_size; - newSpace = ShmemAllocRaw(size, &allocated_size); + newSpace = ShmemAllocRawInSlot(size, &allocated_size, shmem_slot); if (!newSpace) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -184,6 +197,12 @@ ShmemAllocNoError(Size size) */ static void * ShmemAllocRaw(Size size, Size *allocated_size) +{ + return ShmemAllocRawInSlot(size, allocated_size, MAIN_SHMEM_SLOT); +} + +static void * +ShmemAllocRawInSlot(Size size, Size *allocated_size, int shmem_slot) { Size newStart; Size newFree; @@ -203,22 +222,22 @@ ShmemAllocRaw(Size size, Size *allocated_size) size = CACHELINEALIGN(size); *allocated_size = size; - Assert(ShmemSegHdr != NULL); + Assert(Segments[shmem_slot].ShmemSegHdr != NULL); - SpinLockAcquire(ShmemLock); + SpinLockAcquire(Segments[shmem_slot].ShmemLock); - newStart = ShmemSegHdr->freeoffset; + newStart = Segments[shmem_slot].ShmemSegHdr->freeoffset; newFree = newStart + size; - if (newFree <= ShmemSegHdr->totalsize) + if (newFree <= Segments[shmem_slot].ShmemSegHdr->totalsize) { - newSpace = (void *) ((char *) ShmemBase + newStart); - ShmemSegHdr->freeoffset = newFree; + newSpace = (void *) ((char *) Segments[shmem_slot].ShmemBase + newStart); + Segments[shmem_slot].ShmemSegHdr->freeoffset = newFree; } else newSpace = NULL; - SpinLockRelease(ShmemLock); + SpinLockRelease(Segments[shmem_slot].ShmemLock); /* note this assert is okay with newSpace == NULL */ Assert(newSpace == (void *) CACHELINEALIGN(newSpace)); @@ -236,6 +255,12 @@ ShmemAllocRaw(Size size, Size *allocated_size) */ void * ShmemAllocUnlocked(Size size) +{ + return ShmemAllocUnlockedInSlot(size, MAIN_SHMEM_SLOT); +} + +void * +ShmemAllocUnlockedInSlot(Size size, int shmem_slot) { Size newStart; Size newFree; @@ -246,19 +271,19 @@ ShmemAllocUnlocked(Size size) */ size = MAXALIGN(size); - Assert(ShmemSegHdr != NULL); + Assert(Segments[shmem_slot].ShmemSegHdr != NULL); - newStart = ShmemSegHdr->freeoffset; + newStart = Segments[shmem_slot].ShmemSegHdr->freeoffset; newFree = newStart + size; - if (newFree > ShmemSegHdr->totalsize) + if (newFree > Segments[shmem_slot].ShmemSegHdr->totalsize) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of shared memory (%zu bytes requested)", size))); - ShmemSegHdr->freeoffset = newFree; + Segments[shmem_slot].ShmemSegHdr->freeoffset = newFree; - newSpace = (void *) ((char *) ShmemBase + newStart); + newSpace = (void *) ((char *) Segments[shmem_slot].ShmemBase + newStart); Assert(newSpace == (void *) MAXALIGN(newSpace)); @@ -273,7 +298,13 @@ ShmemAllocUnlocked(Size size) bool ShmemAddrIsValid(const void *addr) { - return (addr >= ShmemBase) && (addr < ShmemEnd); + return ShmemAddrIsValidInSlot(addr, MAIN_SHMEM_SLOT); +} + +bool +ShmemAddrIsValidInSlot(const void *addr, int shmem_slot) +{ + return (addr >= Segments[shmem_slot].ShmemBase) && (addr < Segments[shmem_slot].ShmemEnd); } /* @@ -334,6 +365,18 @@ ShmemInitHash(const char *name, /* table string name for shmem index */ long max_size, /* max size of the table */ HASHCTL *infoP, /* info about key and bucket size */ int hash_flags) /* info about infoP */ +{ + return ShmemInitHashInSlot(name, init_size, max_size, infoP, hash_flags, + MAIN_SHMEM_SLOT); +} + +HTAB * +ShmemInitHashInSlot(const char *name, /* table string name for shmem index */ + long init_size, /* initial table size */ + long max_size, /* max size of the table */ + HASHCTL *infoP, /* info about key and bucket size */ + int hash_flags, /* info about infoP */ + int shmem_slot) /* in which slot to keep the table */ { bool found; void *location; @@ -350,9 +393,9 @@ ShmemInitHash(const char *name, /* table string name for shmem index */ hash_flags |= HASH_SHARED_MEM | HASH_ALLOC | HASH_DIRSIZE; /* look it up in the shmem index */ - location = ShmemInitStruct(name, + location = ShmemInitStructInSlot(name, hash_get_shared_size(infoP, hash_flags), - &found); + &found, shmem_slot); /* * if it already exists, attach to it rather than allocate and initialize @@ -385,6 +428,13 @@ ShmemInitHash(const char *name, /* table string name for shmem index */ */ void * ShmemInitStruct(const char *name, Size size, bool *foundPtr) +{ + return ShmemInitStructInSlot(name, size, foundPtr, MAIN_SHMEM_SLOT); +} + +void * +ShmemInitStructInSlot(const char *name, Size size, bool *foundPtr, + int shmem_slot) { ShmemIndexEnt *result; void *structPtr; @@ -393,7 +443,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr) if (!ShmemIndex) { - PGShmemHeader *shmemseghdr = ShmemSegHdr; + PGShmemHeader *shmemseghdr = Segments[shmem_slot].ShmemSegHdr; /* Must be trying to create/attach to ShmemIndex itself */ Assert(strcmp(name, "ShmemIndex") == 0); @@ -416,7 +466,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr) * process can be accessing shared memory yet. */ Assert(shmemseghdr->index == NULL); - structPtr = ShmemAlloc(size); + structPtr = ShmemAllocInSlot(size, shmem_slot); shmemseghdr->index = structPtr; *foundPtr = false; } @@ -433,8 +483,8 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr) LWLockRelease(ShmemIndexLock); ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("could not create ShmemIndex entry for data structure \"%s\"", - name))); + errmsg("could not create ShmemIndex entry for data structure \"%s\" in slot %d", + name, shmem_slot))); } if (*foundPtr) @@ -459,7 +509,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr) Size allocated_size; /* It isn't in the table yet. allocate and initialize it */ - structPtr = ShmemAllocRaw(size, &allocated_size); + structPtr = ShmemAllocRawInSlot(size, &allocated_size, shmem_slot); if (structPtr == NULL) { /* out of memory; remove the failed ShmemIndex entry */ @@ -478,14 +528,13 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr) LWLockRelease(ShmemIndexLock); - Assert(ShmemAddrIsValid(structPtr)); + Assert(ShmemAddrIsValidInSlot(structPtr, shmem_slot)); Assert(structPtr == (void *) CACHELINEALIGN(structPtr)); return structPtr; } - /* * Add two Size values, checking for overflow */ @@ -545,7 +594,7 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS) while ((ent = (ShmemIndexEnt *) hash_seq_search(&hstat)) != NULL) { values[0] = CStringGetTextDatum(ent->key); - values[1] = Int64GetDatum((char *) ent->location - (char *) ShmemSegHdr); + values[1] = Int64GetDatum((char *) ent->location - (char *) Segments[MAIN_SHMEM_SLOT].ShmemSegHdr); values[2] = Int64GetDatum(ent->size); values[3] = Int64GetDatum(ent->allocated_size); named_allocated += ent->allocated_size; @@ -557,15 +606,15 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS) /* output shared memory allocated but not counted via the shmem index */ values[0] = CStringGetTextDatum("<anonymous>"); nulls[1] = true; - values[2] = Int64GetDatum(ShmemSegHdr->freeoffset - named_allocated); + values[2] = Int64GetDatum(Segments[MAIN_SHMEM_SLOT].ShmemSegHdr->freeoffset - named_allocated); values[3] = values[2]; tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); /* output as-of-yet unused shared memory */ nulls[0] = true; - values[1] = Int64GetDatum(ShmemSegHdr->freeoffset); + values[1] = Int64GetDatum(Segments[MAIN_SHMEM_SLOT].ShmemSegHdr->freeoffset); nulls[1] = false; - values[2] = Int64GetDatum(ShmemSegHdr->totalsize - ShmemSegHdr->freeoffset); + values[2] = Int64GetDatum(Segments[MAIN_SHMEM_SLOT].ShmemSegHdr->totalsize - Segments[MAIN_SHMEM_SLOT].ShmemSegHdr->freeoffset); values[3] = values[2]; tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index e765754d80..fb0c33bf17 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -81,6 +81,7 @@ #include "pgstat.h" #include "port/pg_bitutils.h" #include "postmaster/postmaster.h" +#include "storage/pg_shmem.h" #include "storage/proc.h" #include "storage/proclist.h" #include "storage/spin.h" @@ -607,9 +608,9 @@ LWLockNewTrancheId(void) LWLockCounter = (int *) ((char *) MainLWLockArray - sizeof(int)); /* We use the ShmemLock spinlock to protect LWLockCounter */ - SpinLockAcquire(ShmemLock); + SpinLockAcquire(Segments[MAIN_SHMEM_SLOT].ShmemLock); result = (*LWLockCounter)++; - SpinLockRelease(ShmemLock); + SpinLockRelease(Segments[MAIN_SHMEM_SLOT].ShmemLock); return result; } diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index f190e6e5e4..aef80e049b 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -23,6 +23,7 @@ #include "storage/latch.h" #include "storage/lwlock.h" #include "storage/shmem.h" +#include "storage/pg_shmem.h" #include "storage/smgr.h" #include "storage/spin.h" #include "utils/relcache.h" diff --git a/src/include/storage/ipc.h b/src/include/storage/ipc.h index b2d062781e..be4b131288 100644 --- a/src/include/storage/ipc.h +++ b/src/include/storage/ipc.h @@ -77,7 +77,7 @@ extern void check_on_shmem_exit_lists_are_empty(void); /* ipci.c */ extern PGDLLIMPORT shmem_startup_hook_type shmem_startup_hook; -extern Size CalculateShmemSize(int *num_semaphores); +extern Size CalculateShmemSize(int *num_semaphores, int shmem_slot); extern void CreateSharedMemoryAndSemaphores(void); #ifdef EXEC_BACKEND extern void AttachSharedMemoryStructs(void); diff --git a/src/include/storage/pg_sema.h b/src/include/storage/pg_sema.h index dfef79ac96..081fffaf16 100644 --- a/src/include/storage/pg_sema.h +++ b/src/include/storage/pg_sema.h @@ -41,7 +41,7 @@ typedef HANDLE PGSemaphore; extern Size PGSemaphoreShmemSize(int maxSemas); /* Module initialization (called during postmaster start or shmem reinit) */ -extern void PGReserveSemaphores(int maxSemas); +extern void PGReserveSemaphores(int maxSemas, int shmem_slot); /* Allocate a PGSemaphore structure with initial count 1 */ extern PGSemaphore PGSemaphoreCreate(void); diff --git a/src/include/storage/pg_shmem.h b/src/include/storage/pg_shmem.h index 3065ff5be7..e968deeef7 100644 --- a/src/include/storage/pg_shmem.h +++ b/src/include/storage/pg_shmem.h @@ -25,6 +25,7 @@ #define PG_SHMEM_H #include "storage/dsm_impl.h" +#include "storage/spin.h" typedef struct PGShmemHeader /* standard header for all Postgres shmem */ { @@ -41,6 +42,20 @@ typedef struct PGShmemHeader /* standard header for all Postgres shmem */ #endif } PGShmemHeader; +typedef struct ShmemSegment +{ + PGShmemHeader *ShmemSegHdr; /* shared mem segment header */ + void *ShmemBase; /* start address of shared memory */ + void *ShmemEnd; /* end+1 address of shared memory */ + slock_t *ShmemLock; /* spinlock for shared memory and LWLock + * allocation */ +} ShmemSegment; + +// Number of available slots for anonymous memory mappings +#define ANON_MAPPINGS 1 + +extern PGDLLIMPORT ShmemSegment Segments[ANON_MAPPINGS]; + /* GUC variables */ extern PGDLLIMPORT int shared_memory_type; extern PGDLLIMPORT int huge_pages; @@ -90,4 +105,7 @@ extern bool PGSharedMemoryIsInUse(unsigned long id1, unsigned long id2); extern void PGSharedMemoryDetach(void); extern void GetHugePageSize(Size *hugepagesize, int *mmap_flags); +/* The main slot, contains everything except buffer blocks and related data. */ +#define MAIN_SHMEM_SLOT 0 + #endif /* PG_SHMEM_H */ diff --git a/src/include/storage/shmem.h b/src/include/storage/shmem.h index 842989111c..d3e9cc721d 100644 --- a/src/include/storage/shmem.h +++ b/src/include/storage/shmem.h @@ -28,15 +28,25 @@ /* shmem.c */ extern PGDLLIMPORT slock_t *ShmemLock; extern void InitShmemAccess(void *seghdr); +extern void InitShmemAccessInSlot(void *seghdr, int shmem_slot); extern void InitShmemAllocation(void); +extern void InitShmemAllocationInSlot(int shmem_slot); extern void *ShmemAlloc(Size size); +extern void *ShmemAllocInSlot(Size size, int shmem_slot); extern void *ShmemAllocNoError(Size size); extern void *ShmemAllocUnlocked(Size size); +extern void *ShmemAllocUnlockedInSlot(Size size, int shmem_slot); extern bool ShmemAddrIsValid(const void *addr); +extern bool ShmemAddrIsValidInSlot(const void *addr, int shmem_slot); extern void InitShmemIndex(void); +extern void InitVariableShmemIndex(void); extern HTAB *ShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags); +extern HTAB *ShmemInitHashInSlot(const char *name, long init_size, long max_size, + HASHCTL *infoP, int hash_flags, int shmem_slot); extern void *ShmemInitStruct(const char *name, Size size, bool *foundPtr); +extern void *ShmemInitStructInSlot(const char *name, Size size, bool *foundPtr, + int shmem_slot); extern Size add_size(Size s1, Size s2); extern Size mul_size(Size s1, Size s2); base-commit: 2488058dc356a43455b21a099ea879fff9266634 -- 2.45.1 --ers4zrhjqnhlajas Content-Type: text/plain; charset=us-ascii Content-Disposition: attachment; filename="v1-0002-Allow-placing-shared-memory-mapping-with-an-offse.patch" ^ permalink raw reply [nested|flat] 22+ messages in thread
end of thread, other threads:[~2024-10-09 13:41 UTC | newest] Thread overview: 22+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-03-08 22:15 [PATCH v9 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-08 22:15 [PATCH v9 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v37 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v32 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v35 3/7] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v30 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v22 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v15 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v30 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v33 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v15 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v36 3/7] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v31 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v31 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v20 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v21 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v19 04/10] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v34 03/15] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v28 03/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v25 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2020-03-10 03:40 [PATCH v24 04/11] Add pg_ls_dir_metadata to list a dir with file metadata.. Justin Pryzby <pryzbyj@telsasoft.com> 2024-10-09 13:41 [PATCH v1 1/5] Allow to use multiple shared memory mappings Dmitrii Dolgov <9erthalion6@gmail.com>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox