public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v14 6/8] Add pg_ls_dir_recurse to show dir recursively.. 60+ messages / 5 participants [nested] [flat]
* [PATCH v14 6/8] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? XXX: this test case is unstable, if backends/autovacuum remove FSM... ERROR: could not stat file "./base/16384/20345_fsm": No such file or directory CONTEXT: SQL function "pg_ls_dir_recurse" statement 1 Need catversion bumped ? --- doc/src/sgml/func.sgml | 25 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 +++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 52 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4cdd03610a..b30ac9f10d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21351,6 +21351,16 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21462,6 +21472,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +lateral format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +lateral pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 05a644a7c9..e7295d8aaf 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1436,6 +1436,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 2c5829bca4..af2c2dd621 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10753,6 +10753,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Kynn+LdAwU9N+JqL Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v14-0007-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v18 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4aa9cdb18c..683f533392 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index cc8e870740..58d535a9f6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1469,6 +1469,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b61f30247e..bac1226db7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10914,6 +10914,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false) as a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --5I6of5zJg18YgZEa Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v18-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v20 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 13 ++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 58 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9652409581..befd2ffca5 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25741,6 +25741,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 360b6bda26..44690be916 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1479,6 +1479,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0b5716525c..e60b6ca866 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10931,6 +10931,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,text,int8,timestamptz,bool}', + proargnames => '{dirname,path,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select dirname as path, * from pg_ls_dir_metadata(dirname, true, false) union all select parent.path||'/'||parent.name, a.name, a.size, a.modification, a.isdir from ls AS parent, lateral pg_ls_dir_metadata(parent.path||'/'||parent.name, false, false) as a where parent.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..07c0a7f961 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,19 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT path, name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND path='./pg_wal'; + path | name | isdir +----------+----------------+------- + ./pg_wal | archive_status | t +(1 row) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + path | name | size | modification | isdir +------+------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..349752da94 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT path, name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND path='./pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Z1Z8UV8BNhgCynIS Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v20-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v21 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/bin/pg_rewind/libpq_fetch.c | 22 +++----------- src/bin/pg_rewind/t/RewindTest.pm | 7 ++++- src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 13 ++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 7 files changed, 69 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9652409581..befd2ffca5 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25741,6 +25741,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b9b2a6aa20..cdf105ffac 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1479,6 +1479,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index c44648f823..58f2bb9fd1 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -176,30 +176,18 @@ libpqProcessFileList(void) /* * Create a recursive directory listing of the whole data directory. * - * The WITH RECURSIVE part does most of the work. The second part gets the - * targets of the symlinks in pg_tblspc directory. + * Join to pg_tablespace to get the targets of the symlinks in + * pg_tblspc directory. * * XXX: There is no backend function to get a symbolic link's target in * general, so if the admin has put any custom symbolic links in the data * directory, they won't be copied correctly. */ sql = - "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" - " SELECT '' AS path, filename, size, isdir FROM\n" - " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n" - " pg_stat_file(fn.filename, true) AS this\n" - " UNION ALL\n" - " SELECT parent.path || parent.filename || '/' AS path,\n" - " fn, this.size, this.isdir\n" - " FROM files AS parent,\n" - " pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n" - " pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n" - " WHERE parent.isdir = 't'\n" - ")\n" - "SELECT path || filename, size, isdir,\n" + "SELECT COALESCE(NULLIF(path,'.')||'/','')||filename, size, isdir,\n" " pg_tablespace_location(pg_tablespace.oid) AS link_target\n" - "FROM files\n" - "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" + "FROM pg_ls_dir_recurse('.') files\n" + "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc'\n" " AND oid::text = files.filename\n"; res = PQexec(conn, sql); diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm index 7516af7a01..bc8dad074e 100644 --- a/src/bin/pg_rewind/t/RewindTest.pm +++ b/src/bin/pg_rewind/t/RewindTest.pm @@ -160,7 +160,12 @@ sub start_primary GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO rewind_user; GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) - TO rewind_user;"); + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_metadata(text, bool, bool) + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_recurse(text) + TO rewind_user; + "); #### Now run the test-specific parts to initialize the primary before setting # up standby diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 58d1c74b52..826a31c561 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10931,6 +10931,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,filename,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,text,int8,timestamptz,bool}', + proargnames => '{dirname,path,filename,size,modification,isdir}', proargmodes => '{i,o,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select dirname as path, * from pg_ls_dir_metadata(dirname, false, false) union all select coalesce(nullif(parent.path,'.')||'/','')||parent.filename, a.filename, a.size, a.modification, a.isdir from ls as parent, lateral pg_ls_dir_metadata(parent.path||'/'||parent.filename, false, false) as a where parent.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index bb926a2cf4..302f348a6d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,19 @@ select * from pg_ls_dir_metadata('.') limit 0; ----------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + path | filename | isdir +--------+----------------+------- + pg_wal | archive_status | t +(1 row) + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + path | filename | size | modification | isdir +------+----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 68e2bc6586..fda95828d7 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Tcb1KvpfnM4LxW2s Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v21-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v9 07/11] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? TODO: src/backend/catalog/system_views.sql:REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; Need catversion bumped ? --- doc/src/sgml/func.sgml | 13 +++++++++++++ src/include/catalog/pg_proc.dat | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7c73d3b82a..672cbab7b9 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21442,6 +21442,19 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + <parameter>missing_ok</parameter> ??? + To recusively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp')dir FROM pg_tablespace b, pg_control_system() pcs, format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) suffix)dir, pg_ls_dir_recurse(dir)a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7789d029ea..cc2c6f6571 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -6143,6 +6143,14 @@ proname => 'pg_ls_dir', prorows => '1000', proretset => 't', provolatile => 'v', prorettype => 'text', proargtypes => 'text bool bool', prosrc => 'pg_ls_dir' }, + +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "WITH RECURSIVE x AS (SELECT * FROM pg_ls_dir_metadata(dirname, true, false, true) UNION ALL SELECT x.name||'/'||a.name, a.size, a.modification, a.isdir FROM x, pg_ls_dir_metadata(dirname||'/'||x.name, true, false, true)a WHERE x.isdir) SELECT * FROM x" }, + { oid => '2626', descr => 'sleep for the specified time in seconds', proname => 'pg_sleep', provolatile => 'v', prorettype => 'void', proargtypes => 'float8', prosrc => 'pg_sleep' }, -- 2.17.0 --32u276st3Jlj2kUU Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0008-generalize-pg_ls_dir_files-and-retire-pg_ls_dir.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v10 7/9] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? TODO: src/backend/catalog/system_views.sql:REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; Need catversion bumped ? --- doc/src/sgml/func.sgml | 13 +++++++++++++ src/include/catalog/pg_proc.dat | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 3d8f7dff96..d0b782d803 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21445,6 +21445,19 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + <parameter>missing_ok</parameter> ??? + To recusively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp')dir FROM pg_tablespace b, pg_control_system() pcs, format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) suffix)dir, pg_ls_dir_recurse(dir)a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b4c3d15495..75423db22e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -6144,6 +6144,13 @@ provolatile => 'v', prorettype => 'text', proargtypes => 'text bool bool', prosrc => 'pg_ls_dir' }, +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "WITH RECURSIVE x AS (SELECT * FROM pg_ls_dir_metadata(dirname, true, false, true) UNION ALL SELECT x.name||'/'||a.name, a.size, a.modification, a.isdir FROM x, pg_ls_dir_metadata(dirname||'/'||x.name, true, false, true)a WHERE x.isdir) SELECT * FROM x" }, + { oid => '2626', descr => 'sleep for the specified time in seconds', proname => 'pg_sleep', provolatile => 'v', prorettype => 'void', proargtypes => 'float8', prosrc => 'pg_sleep' }, -- 2.17.0 --rCwQ2Y43eQY6RBgR Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v11 7/9] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? TODO: src/backend/catalog/system_views.sql:REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; Need catversion bumped ? --- doc/src/sgml/func.sgml | 24 ++++++++++++++++++++ src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 4 files changed, 50 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index e4ba3fee40..f3422808d9 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21351,6 +21351,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified directory, along with each file's metadata. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21462,6 +21471,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) AS suffix) AS dir, +pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b4c3d15495..667ea5721e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10748,6 +10748,12 @@ proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 448901d841..82d967a62d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -187,6 +187,20 @@ select * from pg_ls_tmpdir() where name='Does not exist'; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 656aad93e6..2cfdac386d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -55,6 +55,12 @@ select count(*) >= 0 as ok from pg_ls_archive_statusdir(); -- The name='' condition is never true, so the function runs to completion but returns zero rows. select * from pg_ls_tmpdir() where name='Does not exist'; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --oTHb8nViIGeoXxdp Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v11-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v12 09/11] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? TODO: src/backend/catalog/system_views.sql:REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; Need catversion bumped ? --- doc/src/sgml/func.sgml | 24 ++++++++++++++++++++ src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 4 files changed, 50 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4cdd03610a..373e3b647b 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21351,6 +21351,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified directory, along with each file's metadata. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21462,6 +21471,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) AS suffix) AS dir, +pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b4c3d15495..667ea5721e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10748,6 +10748,12 @@ proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 448901d841..82d967a62d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -187,6 +187,20 @@ select * from pg_ls_tmpdir() where name='Does not exist'; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 656aad93e6..2cfdac386d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -55,6 +55,12 @@ select count(*) >= 0 as ok from pg_ls_archive_statusdir(); -- The name='' condition is never true, so the function runs to completion but returns zero rows. select * from pg_ls_tmpdir() where name='Does not exist'; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --wIc/V6YLA2QdyfT4 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0010-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v13 6/8] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? XXX: this test case is unstable, if backends/autovacuum remove FSM... ERROR: could not stat file "./base/16384/20345_fsm": No such file or directory CONTEXT: SQL function "pg_ls_dir_recurse" statement 1 Need catversion bumped ? --- doc/src/sgml/func.sgml | 25 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 +++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 52 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4cdd03610a..0763db3f50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21351,6 +21351,16 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21462,6 +21472,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) AS suffix) AS dir, +pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 05a644a7c9..e7295d8aaf 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1436,6 +1436,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index be3e820a03..b3509f0e12 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10747,6 +10747,12 @@ proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 3ca4b85dd9..ae8716a83f 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -208,6 +208,20 @@ select * from pg_ls_tmpdir() where name='Does not exist'; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f09c890b5d..7038bcdb0f 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -64,6 +64,12 @@ select count(*) > 0 from -- The name='' condition is never true, so the function runs to completion but returns zero rows. select * from pg_ls_tmpdir() where name='Does not exist'; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --E0h0CbphJD8hN+Gf Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0007-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v22 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/bin/pg_rewind/libpq_fetch.c | 22 +++----------- src/bin/pg_rewind/t/RewindTest.pm | 7 ++++- src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 13 ++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 7 files changed, 69 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index f7ccf68ff4..7baf2b31e5 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25769,6 +25769,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 169602c1b6..55b700dadc 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1510,6 +1510,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index bf4dfc23b9..a9535d3dd5 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -176,30 +176,18 @@ libpqProcessFileList(void) /* * Create a recursive directory listing of the whole data directory. * - * The WITH RECURSIVE part does most of the work. The second part gets the - * targets of the symlinks in pg_tblspc directory. + * Join to pg_tablespace to get the targets of the symlinks in + * pg_tblspc directory. * * XXX: There is no backend function to get a symbolic link's target in * general, so if the admin has put any custom symbolic links in the data * directory, they won't be copied correctly. */ sql = - "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" - " SELECT '' AS path, filename, size, isdir FROM\n" - " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n" - " pg_stat_file(fn.filename, true) AS this\n" - " UNION ALL\n" - " SELECT parent.path || parent.filename || '/' AS path,\n" - " fn, this.size, this.isdir\n" - " FROM files AS parent,\n" - " pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n" - " pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n" - " WHERE parent.isdir = 't'\n" - ")\n" - "SELECT path || filename, size, isdir,\n" + "SELECT COALESCE(NULLIF(path,'.')||'/','')||filename, size, isdir,\n" " pg_tablespace_location(pg_tablespace.oid) AS link_target\n" - "FROM files\n" - "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" + "FROM pg_ls_dir_recurse('.') files\n" + "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc'\n" " AND oid::text = files.filename\n"; res = PQexec(conn, sql); diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm index 41ed7d4b3b..cacd58651c 100644 --- a/src/bin/pg_rewind/t/RewindTest.pm +++ b/src/bin/pg_rewind/t/RewindTest.pm @@ -160,7 +160,12 @@ sub start_primary GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO rewind_user; GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) - TO rewind_user;"); + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_metadata(text, bool, bool) + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_recurse(text) + TO rewind_user; + "); #### Now run the test-specific parts to initialize the primary before setting # up standby diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index c3564168e5..0235042e26 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10976,6 +10976,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,filename,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,text,int8,timestamptz,bool}', + proargnames => '{dirname,path,filename,size,modification,isdir}', proargmodes => '{i,o,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select dirname as path, * from pg_ls_dir_metadata(dirname, false, false) union all select coalesce(nullif(parent.path,'.')||'/','')||parent.filename, a.filename, a.size, a.modification, a.isdir from ls as parent, lateral pg_ls_dir_metadata(parent.path||'/'||parent.filename, false, false) as a where parent.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index bb926a2cf4..302f348a6d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,19 @@ select * from pg_ls_dir_metadata('.') limit 0; ----------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + path | filename | isdir +--------+----------------+------- + pg_wal | archive_status | t +(1 row) + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + path | filename | size | modification | isdir +------+----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 68e2bc6586..fda95828d7 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --d6Gm4EdcadzBjdND Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v22-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v23 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/bin/pg_rewind/libpq_source.c | 22 +++----------- src/bin/pg_rewind/t/RewindTest.pm | 7 ++++- src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 13 ++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 7 files changed, 69 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 729d201249..c95dab6e1d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25760,6 +25760,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index e7647787cf..148dd7a8a8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1504,6 +1504,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/bin/pg_rewind/libpq_source.c b/src/bin/pg_rewind/libpq_source.c index c73e8bf470..46c8b7fcd5 100644 --- a/src/bin/pg_rewind/libpq_source.c +++ b/src/bin/pg_rewind/libpq_source.c @@ -205,30 +205,18 @@ libpq_traverse_files(rewind_source *source, process_file_callback_t callback) /* * Create a recursive directory listing of the whole data directory. * - * The WITH RECURSIVE part does most of the work. The second part gets the - * targets of the symlinks in pg_tblspc directory. + * Join to pg_tablespace to get the targets of the symlinks in + * pg_tblspc directory. * * XXX: There is no backend function to get a symbolic link's target in * general, so if the admin has put any custom symbolic links in the data * directory, they won't be copied correctly. */ sql = - "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" - " SELECT '' AS path, filename, size, isdir FROM\n" - " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n" - " pg_stat_file(fn.filename, true) AS this\n" - " UNION ALL\n" - " SELECT parent.path || parent.filename || '/' AS path,\n" - " fn, this.size, this.isdir\n" - " FROM files AS parent,\n" - " pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n" - " pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n" - " WHERE parent.isdir = 't'\n" - ")\n" - "SELECT path || filename, size, isdir,\n" + "SELECT COALESCE(NULLIF(path,'.')||'/','')||filename, size, isdir,\n" " pg_tablespace_location(pg_tablespace.oid) AS link_target\n" - "FROM files\n" - "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" + "FROM pg_ls_dir_recurse('.') files\n" + "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc'\n" " AND oid::text = files.filename\n"; res = PQexec(conn, sql); diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm index 41ed7d4b3b..cacd58651c 100644 --- a/src/bin/pg_rewind/t/RewindTest.pm +++ b/src/bin/pg_rewind/t/RewindTest.pm @@ -160,7 +160,12 @@ sub start_primary GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO rewind_user; GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) - TO rewind_user;"); + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_metadata(text, bool, bool) + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_recurse(text) + TO rewind_user; + "); #### Now run the test-specific parts to initialize the primary before setting # up standby diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 344d6305e5..5e8f16d5b1 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10983,6 +10983,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,filename,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,text,int8,timestamptz,bool}', + proargnames => '{dirname,path,filename,size,modification,isdir}', proargmodes => '{i,o,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select dirname as path, * from pg_ls_dir_metadata(dirname, false, false) union all select coalesce(nullif(parent.path,'.')||'/','')||parent.filename, a.filename, a.size, a.modification, a.isdir from ls as parent, lateral pg_ls_dir_metadata(parent.path||'/'||parent.filename, false, false) as a where parent.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index bb926a2cf4..302f348a6d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,19 @@ select * from pg_ls_dir_metadata('.') limit 0; ----------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + path | filename | isdir +--------+----------------+------- + pg_wal | archive_status | t +(1 row) + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + path | filename | size | modification | isdir +------+----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 68e2bc6586..fda95828d7 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --mhjHhnbe5PrRcwjY Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v23-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v19 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d1d2f868d8..c097977d9b 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25733,6 +25733,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index f5217a0d93..297b0dcdb8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1482,6 +1482,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b61f30247e..bac1226db7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10914,6 +10914,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false) as a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --S0GG+JvAI2G0KxBG Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v19-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 60+ messages in thread
* [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS @ 2026-04-25 19:52 SATYANARAYANA NARLAPURAM <[email protected]> 0 siblings, 1 reply; 60+ messages in thread From: SATYANARAYANA NARLAPURAM @ 2026-04-25 19:52 UTC (permalink / raw) To: PostgreSQL Hackers <[email protected]>; Ashutosh Bapat <[email protected]> Hi hackers, transformRangeGraphTable() calls transformExpr() and assign_list_collations() for COLUMNS expressions but missed calling resolveTargetListUnknowns(). As a result, literals such as 'val1' in a COLUMNS clause retained type "unknown", causing failures with ORDER BY, UNION, and output conversions. Fix by calling resolveTargetListUnknowns() on the columns target list right after assign_list_collations(), similar to SELECT target lists in transformSelectStmt(). Attached a patch to fix this, which also includes test cases to reproduce. Thanks, Satya Attachments: [application/octet-stream] 0001-Resolve-unknown-type-literals-in-GRAPH_TABLE-COLUMNS.patch (4.3K, ../../CAHg+QDcyKNWyzDoKMxiZNjv7C-wAxs8y0ZoNkOV137Y+nk3UXg@mail.gmail.com/3-0001-Resolve-unknown-type-literals-in-GRAPH_TABLE-COLUMNS.patch) download | inline diff: From efa000905343f6c2512f363c0013c19fbad5b8a5 Mon Sep 17 00:00:00 2001 From: Satya Narlapuram <[email protected]> Date: Sat, 25 Apr 2026 17:30:43 +0000 Subject: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS expressions transformRangeGraphTable() calls transformExpr() and assign_list_collations() for COLUMNS expressions but missed calling resolveTargetListUnknowns(). As a result, literals such as 'col1' in a COLUMNS clause retained type "unknown", causing failures with ORDER BY, UNION, and output conversions. Fix by calling resolveTargetListUnknowns() on the columns target list right after assign_list_collations(), similar to SELECT target lists in transformSelectStmt(). --- src/backend/parser/parse_clause.c | 3 ++ src/test/regress/expected/graph_table.out | 54 +++++++++++++++++++++++ src/test/regress/sql/graph_table.sql | 21 +++++++++ 3 files changed, 78 insertions(+) diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c index f3736979..77981850 100644 --- a/src/backend/parser/parse_clause.c +++ b/src/backend/parser/parse_clause.c @@ -1010,6 +1010,9 @@ transformRangeGraphTable(ParseState *pstate, RangeGraphTable *rgt) */ assign_list_collations(pstate, columns); + /* Resolve unknown-type literals to text */ + resolveTargetListUnknowns(pstate, columns); + table_close(rel, NoLock); pstate->p_graph_table_pstate = NULL; diff --git a/src/test/regress/expected/graph_table.out b/src/test/regress/expected/graph_table.out index d3c20610..212fa29f 100644 --- a/src/test/regress/expected/graph_table.out +++ b/src/test/regress/expected/graph_table.out @@ -1110,4 +1110,58 @@ SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) COLUMNS (upper(a.vname) AS n)) OR V13 (3 rows) +-- String and NULL literals in COLUMNS must resolve to proper types +-- (not remain as "unknown") +SELECT pg_typeof(c1), c1 + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) ORDER BY n; + pg_typeof | c1 +-----------+------ + text | val1 + text | val1 + text | val1 +(3 rows) + +-- ORDER BY on a literal column +SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) ORDER BY c1, n; + n | c1 +-----+------ + v11 | val1 + v12 | val1 + v13 | val1 +(3 rows) + +-- UNION between GRAPH_TABLE literal column and plain text +SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) + UNION ALL SELECT 'val2', 'val3'; + n | c1 +------+------ + v11 | val1 + v12 | val1 + v13 | val1 + val2 | val3 +(4 rows) + +-- NULL literal must not cause failures +SELECT n, pg_typeof(c1) + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, NULL AS c1)) ORDER BY n; + n | pg_typeof +-----+----------- + v11 | text + v12 | text + v13 | text +(3 rows) + +-- integer literal type is preserved +SELECT pg_typeof(c1) + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (42 AS c1)) LIMIT 1; + pg_typeof +----------- + integer +(1 row) + -- leave the objects behind for pg_upgrade/pg_dump tests diff --git a/src/test/regress/sql/graph_table.sql b/src/test/regress/sql/graph_table.sql index 2521c4ca..3d74ab4c 100644 --- a/src/test/regress/sql/graph_table.sql +++ b/src/test/regress/sql/graph_table.sql @@ -634,4 +634,25 @@ SELECT src.vname, count(*) SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) COLUMNS (a.vname || '!' AS n)) ORDER BY n; SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) COLUMNS (upper(a.vname) AS n)) ORDER BY n; +-- String and NULL literals in COLUMNS must resolve to proper types +-- (not remain as "unknown") +SELECT pg_typeof(c1), c1 + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) ORDER BY n; +-- ORDER BY on a literal column +SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) ORDER BY c1, n; +-- UNION between GRAPH_TABLE literal column and plain text +SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) + UNION ALL SELECT 'val2', 'val3'; +-- NULL literal must not cause failures +SELECT n, pg_typeof(c1) + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, NULL AS c1)) ORDER BY n; +-- integer literal type is preserved +SELECT pg_typeof(c1) + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (42 AS c1)) LIMIT 1; + -- leave the objects behind for pg_upgrade/pg_dump tests -- 2.43.0 ^ permalink raw reply [nested|flat] 60+ messages in thread
* Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS @ 2026-04-26 15:10 Junwang Zhao <[email protected]> parent: SATYANARAYANA NARLAPURAM <[email protected]> 0 siblings, 1 reply; 60+ messages in thread From: Junwang Zhao @ 2026-04-26 15:10 UTC (permalink / raw) To: SATYANARAYANA NARLAPURAM <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Ashutosh Bapat <[email protected]> Hi SATYANARAYANA, On Sun, Apr 26, 2026 at 3:53 AM SATYANARAYANA NARLAPURAM <[email protected]> wrote: > > Hi hackers, > > transformRangeGraphTable() calls transformExpr() and > assign_list_collations() for COLUMNS expressions but missed calling > resolveTargetListUnknowns(). As a result, literals such as 'val1' > in a COLUMNS clause retained type "unknown", causing failures with > ORDER BY, UNION, and output conversions. > > Fix by calling resolveTargetListUnknowns() on the columns target > list right after assign_list_collations(), similar to SELECT target lists in > transformSelectStmt(). > > Attached a patch to fix this, which also includes test cases to reproduce. I can reproduce this and the patch fixes it. One question: why is resolveTargetListUnknowns called after assign_list_collations? I'm asking because in transformSelectStmt, resolveTargetListUnknowns is invoked before assign_query_collations. It might not matter, but keeping the order consistent would be good for readers. > > > Thanks, > Satya > > -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 60+ messages in thread
* Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS @ 2026-04-27 06:04 SATYANARAYANA NARLAPURAM <[email protected]> parent: Junwang Zhao <[email protected]> 0 siblings, 1 reply; 60+ messages in thread From: SATYANARAYANA NARLAPURAM @ 2026-04-27 06:04 UTC (permalink / raw) To: Junwang Zhao <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Ashutosh Bapat <[email protected]> Hi, On Sun, Apr 26, 2026 at 8:10 AM Junwang Zhao <[email protected]> wrote: > Hi SATYANARAYANA, > > On Sun, Apr 26, 2026 at 3:53 AM SATYANARAYANA NARLAPURAM > <[email protected]> wrote: > > > > Hi hackers, > > > > transformRangeGraphTable() calls transformExpr() and > > assign_list_collations() for COLUMNS expressions but missed calling > > resolveTargetListUnknowns(). As a result, literals such as 'val1' > > in a COLUMNS clause retained type "unknown", causing failures with > > ORDER BY, UNION, and output conversions. > > > > Fix by calling resolveTargetListUnknowns() on the columns target > > list right after assign_list_collations(), similar to SELECT target > lists in > > transformSelectStmt(). > > > > Attached a patch to fix this, which also includes test cases to > reproduce. > > I can reproduce this and the patch fixes it. > > One question: why is resolveTargetListUnknowns called after > assign_list_collations? > > I'm asking because in transformSelectStmt, resolveTargetListUnknowns > is invoked before assign_query_collations. It might not matter, but keeping > the order consistent would be good for readers. Updated the patch. It should be before. Thanks, Satya Attachments: [application/octet-stream] v2-0001-Resolve-unknown-type-literals-in-GRAPH_TABLE-COLUMNS.patch (4.1K, ../../CAHg+QDdzUMW4qQmywiPmVo2z3+HccFZL+mujdXbB+ghU21u7dA@mail.gmail.com/3-v2-0001-Resolve-unknown-type-literals-in-GRAPH_TABLE-COLUMNS.patch) download | inline diff: From: Satya Narlapuram <[email protected]> Subject: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS expressions transformRangeGraphTable() calls transformExpr() and assign_list_collations() for COLUMNS expressions but missed calling resolveTargetListUnknowns(). As a result, literals such as 'col1' in a COLUMNS clause retained type "unknown", causing failures with ORDER BY, UNION, and output conversions. Fix by calling resolveTargetListUnknowns() on the columns target list right before assign_list_collations(), similar to SELECT target lists in transformSelectStmt(). --- diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c index f3736979..f679f510 100644 --- a/src/backend/parser/parse_clause.c +++ b/src/backend/parser/parse_clause.c @@ -1004,6 +1004,9 @@ transformRangeGraphTable(ParseState *pstate, RangeGraphTable *rgt) columns = lappend(columns, te); } + /* Resolve unknown-type literals to text */ + resolveTargetListUnknowns(pstate, columns); + /* * Assign collations to column expressions now since * assign_query_collations() does not process rangetable entries. diff --git a/src/test/regress/expected/graph_table.out b/src/test/regress/expected/graph_table.out index d3c20610..212fa29f 100644 --- a/src/test/regress/expected/graph_table.out +++ b/src/test/regress/expected/graph_table.out @@ -1110,4 +1110,58 @@ SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) COLUMNS (upper(a.vname) AS n)) OR V13 (3 rows) +-- String and NULL literals in COLUMNS must resolve to proper types +-- (not remain as "unknown") +SELECT pg_typeof(c1), c1 + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) ORDER BY n; + pg_typeof | c1 +-----------+------ + text | val1 + text | val1 + text | val1 +(3 rows) + +-- ORDER BY on a literal column +SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) ORDER BY c1, n; + n | c1 +-----+------ + v11 | val1 + v12 | val1 + v13 | val1 +(3 rows) + +-- UNION between GRAPH_TABLE literal column and plain text +SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) + UNION ALL SELECT 'val2', 'val3'; + n | c1 +------+------ + v11 | val1 + v12 | val1 + v13 | val1 + val2 | val3 +(4 rows) + +-- NULL literal must not cause failures +SELECT n, pg_typeof(c1) + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, NULL AS c1)) ORDER BY n; + n | pg_typeof +-----+----------- + v11 | text + v12 | text + v13 | text +(3 rows) + +-- integer literal type is preserved +SELECT pg_typeof(c1) + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (42 AS c1)) LIMIT 1; + pg_typeof +----------- + integer +(1 row) + -- leave the objects behind for pg_upgrade/pg_dump tests diff --git a/src/test/regress/sql/graph_table.sql b/src/test/regress/sql/graph_table.sql index 2521c4ca..3d74ab4c 100644 --- a/src/test/regress/sql/graph_table.sql +++ b/src/test/regress/sql/graph_table.sql @@ -634,4 +634,25 @@ SELECT src.vname, count(*) SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) COLUMNS (a.vname || '!' AS n)) ORDER BY n; SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) COLUMNS (upper(a.vname) AS n)) ORDER BY n; +-- String and NULL literals in COLUMNS must resolve to proper types +-- (not remain as "unknown") +SELECT pg_typeof(c1), c1 + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) ORDER BY n; +-- ORDER BY on a literal column +SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) ORDER BY c1, n; +-- UNION between GRAPH_TABLE literal column and plain text +SELECT * FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, 'val1' AS c1)) + UNION ALL SELECT 'val2', 'val3'; +-- NULL literal must not cause failures +SELECT n, pg_typeof(c1) + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (a.vname AS n, NULL AS c1)) ORDER BY n; +-- integer literal type is preserved +SELECT pg_typeof(c1) + FROM GRAPH_TABLE (g1 MATCH (a IS vl1) + COLUMNS (42 AS c1)) LIMIT 1; + -- leave the objects behind for pg_upgrade/pg_dump tests ^ permalink raw reply [nested|flat] 60+ messages in thread
* Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS @ 2026-04-29 14:09 Ashutosh Bapat <[email protected]> parent: SATYANARAYANA NARLAPURAM <[email protected]> 0 siblings, 1 reply; 60+ messages in thread From: Ashutosh Bapat @ 2026-04-29 14:09 UTC (permalink / raw) To: SATYANARAYANA NARLAPURAM <[email protected]>; +Cc: Junwang Zhao <[email protected]>; PostgreSQL Hackers <[email protected]> On Mon, Apr 27, 2026 at 11:34 AM SATYANARAYANA NARLAPURAM <[email protected]> wrote: > > Hi, > > On Sun, Apr 26, 2026 at 8:10 AM Junwang Zhao <[email protected]> wrote: >> >> Hi SATYANARAYANA, >> >> On Sun, Apr 26, 2026 at 3:53 AM SATYANARAYANA NARLAPURAM >> <[email protected]> wrote: >> > >> > Hi hackers, >> > >> > transformRangeGraphTable() calls transformExpr() and >> > assign_list_collations() for COLUMNS expressions but missed calling >> > resolveTargetListUnknowns(). As a result, literals such as 'val1' >> > in a COLUMNS clause retained type "unknown", causing failures with >> > ORDER BY, UNION, and output conversions. >> > >> > Fix by calling resolveTargetListUnknowns() on the columns target >> > list right after assign_list_collations(), similar to SELECT target lists in >> > transformSelectStmt(). >> > >> > Attached a patch to fix this, which also includes test cases to reproduce. >> >> I can reproduce this and the patch fixes it. >> >> One question: why is resolveTargetListUnknowns called after >> assign_list_collations? >> >> I'm asking because in transformSelectStmt, resolveTargetListUnknowns >> is invoked before assign_query_collations. It might not matter, but keeping >> the order consistent would be good for readers. > > > Updated the patch. It should be before. Do we really need a test for ORDER BY on a literal column? I replaced all the test queries with a single one which covers all the scenarios covered by those queries. The patch needed to consider pstate->p_resolve_unknowns. Unknown literals are not resolved as text always. See the test query. -- Best Wishes, Ashutosh Bapat Attachments: [text/x-patch] v20260429-0001-Resolve-unknown-type-literals-in-GRAPH_TAB.patch (3.0K, ../../CAExHW5vmmAS6Vroh8W9+vmuM2ReocODRCWPqYozGSyhxCxxL+Q@mail.gmail.com/2-v20260429-0001-Resolve-unknown-type-literals-in-GRAPH_TAB.patch) download | inline diff: From c9e6b0df945e90bb0be43c88d1c55e1ee0a9a982 Mon Sep 17 00:00:00 2001 From: Ashutosh Bapat <[email protected]> Date: Wed, 29 Apr 2026 19:07:13 +0530 Subject: [PATCH v20260429 5/5] Resolve unknown-type literals in GRAPH_TABLE COLUMNS The unknown-type literals in the COLUMNS clause of a GRAPH_TABLE are now resolved to the appropriate types, causing various failures. Call resolveTargetListUnknowns() on the columns targetlist to resolve unknonw type literals. Do this before assigning collations so that the resolved types are used for collations assignment. Reported by: Satya Narlapuram <[email protected]> Author: Satya Narlapuram <[email protected]> Author: Ashutosh Bapat <[email protected]> Reviewed by: Junwang Zhao <[email protected]> Discussion: https://www.postgresql.org/message-id/CAHg+QDcyKNWyzDoKMxiZNjv7C-wAxs8y0ZoNkOV137Y+nk3UXg@mail.gmail.com --- src/backend/parser/parse_clause.c | 4 ++++ src/test/regress/expected/graph_table.out | 11 +++++++++++ src/test/regress/sql/graph_table.sql | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c index 4270c2382c4..3d0585b8efb 100644 --- a/src/backend/parser/parse_clause.c +++ b/src/backend/parser/parse_clause.c @@ -1003,6 +1003,10 @@ transformRangeGraphTable(ParseState *pstate, RangeGraphTable *rgt) columns = lappend(columns, te); } + /* resolve any still-unresolved output columns as being type text */ + if (pstate->p_resolve_unknowns) + resolveTargetListUnknowns(pstate, columns); + /* * Assign collations to column expressions now since * assign_query_collations() does not process rangetable entries. diff --git a/src/test/regress/expected/graph_table.out b/src/test/regress/expected/graph_table.out index 8038fcd39b7..5ef98e281e6 100644 --- a/src/test/regress/expected/graph_table.out +++ b/src/test/regress/expected/graph_table.out @@ -1047,4 +1047,15 @@ SELECT src.vname, count(*) v11 | 1 (3 rows) +-- Unknown type resolution +SELECT c1, pg_typeof(c1), "null", pg_typeof("null") FROM + (SELECT *FROM GRAPH_TABLE (g1 MATCH (a IS vl1) COLUMNS ('1' AS c1, NULL as "null")) + UNION + SELECT 2, NULL); + c1 | pg_typeof | null | pg_typeof +----+-----------+------+----------- + 1 | integer | | text + 2 | integer | | text +(2 rows) + -- leave the objects behind for pg_upgrade/pg_dump tests diff --git a/src/test/regress/sql/graph_table.sql b/src/test/regress/sql/graph_table.sql index a3681c6c0ef..d3ddcdc8e20 100644 --- a/src/test/regress/sql/graph_table.sql +++ b/src/test/regress/sql/graph_table.sql @@ -599,4 +599,10 @@ SELECT src.vname, count(*) COLUMNS (a.vname AS n)) WHERE n = src.vname); +-- Unknown type resolution +SELECT c1, pg_typeof(c1), "null", pg_typeof("null") FROM + (SELECT *FROM GRAPH_TABLE (g1 MATCH (a IS vl1) COLUMNS ('1' AS c1, NULL as "null")) + UNION + SELECT 2, NULL); + -- leave the objects behind for pg_upgrade/pg_dump tests -- 2.34.1 ^ permalink raw reply [nested|flat] 60+ messages in thread
* Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS @ 2026-05-04 15:12 Peter Eisentraut <[email protected]> parent: Ashutosh Bapat <[email protected]> 0 siblings, 1 reply; 60+ messages in thread From: Peter Eisentraut @ 2026-05-04 15:12 UTC (permalink / raw) To: Ashutosh Bapat <[email protected]>; SATYANARAYANA NARLAPURAM <[email protected]>; +Cc: Junwang Zhao <[email protected]>; PostgreSQL Hackers <[email protected]> On 29.04.26 16:09, Ashutosh Bapat wrote: > On Mon, Apr 27, 2026 at 11:34 AM SATYANARAYANA NARLAPURAM > <[email protected]> wrote: >> >> Hi, >> >> On Sun, Apr 26, 2026 at 8:10 AM Junwang Zhao <[email protected]> wrote: >>> >>> Hi SATYANARAYANA, >>> >>> On Sun, Apr 26, 2026 at 3:53 AM SATYANARAYANA NARLAPURAM >>> <[email protected]> wrote: >>>> >>>> Hi hackers, >>>> >>>> transformRangeGraphTable() calls transformExpr() and >>>> assign_list_collations() for COLUMNS expressions but missed calling >>>> resolveTargetListUnknowns(). As a result, literals such as 'val1' >>>> in a COLUMNS clause retained type "unknown", causing failures with >>>> ORDER BY, UNION, and output conversions. >>>> >>>> Fix by calling resolveTargetListUnknowns() on the columns target >>>> list right after assign_list_collations(), similar to SELECT target lists in >>>> transformSelectStmt(). >>>> >>>> Attached a patch to fix this, which also includes test cases to reproduce. >>> >>> I can reproduce this and the patch fixes it. >>> >>> One question: why is resolveTargetListUnknowns called after >>> assign_list_collations? >>> >>> I'm asking because in transformSelectStmt, resolveTargetListUnknowns >>> is invoked before assign_query_collations. It might not matter, but keeping >>> the order consistent would be good for readers. >> >> >> Updated the patch. It should be before. > > Do we really need a test for ORDER BY on a literal column? I replaced > all the test queries with a single one which covers all the scenarios > covered by those queries. > > The patch needed to consider pstate->p_resolve_unknowns. Unknown > literals are not resolved as text always. See the test query. I couldn't find a commit to apply this patch cleanly (the subject says patch 5/5, so maybe you had some unpublished local changes?). After applying the test case manually, it looks like the test output is already correct without the code change. So if this patch is still required, we need a better test case. ^ permalink raw reply [nested|flat] 60+ messages in thread
* Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS @ 2026-05-12 06:20 Ashutosh Bapat <[email protected]> parent: Peter Eisentraut <[email protected]> 0 siblings, 1 reply; 60+ messages in thread From: Ashutosh Bapat @ 2026-05-12 06:20 UTC (permalink / raw) To: Peter Eisentraut <[email protected]>; +Cc: SATYANARAYANA NARLAPURAM <[email protected]>; Junwang Zhao <[email protected]>; PostgreSQL Hackers <[email protected]> On Mon, May 4, 2026 at 8:42 PM Peter Eisentraut <[email protected]> wrote: > > On 29.04.26 16:09, Ashutosh Bapat wrote: > > On Mon, Apr 27, 2026 at 11:34 AM SATYANARAYANA NARLAPURAM > > <[email protected]> wrote: > >> > >> Hi, > >> > >> On Sun, Apr 26, 2026 at 8:10 AM Junwang Zhao <[email protected]> wrote: > >>> > >>> Hi SATYANARAYANA, > >>> > >>> On Sun, Apr 26, 2026 at 3:53 AM SATYANARAYANA NARLAPURAM > >>> <[email protected]> wrote: > >>>> > >>>> Hi hackers, > >>>> > >>>> transformRangeGraphTable() calls transformExpr() and > >>>> assign_list_collations() for COLUMNS expressions but missed calling > >>>> resolveTargetListUnknowns(). As a result, literals such as 'val1' > >>>> in a COLUMNS clause retained type "unknown", causing failures with > >>>> ORDER BY, UNION, and output conversions. > >>>> > >>>> Fix by calling resolveTargetListUnknowns() on the columns target > >>>> list right after assign_list_collations(), similar to SELECT target lists in > >>>> transformSelectStmt(). > >>>> > >>>> Attached a patch to fix this, which also includes test cases to reproduce. > >>> > >>> I can reproduce this and the patch fixes it. > >>> > >>> One question: why is resolveTargetListUnknowns called after > >>> assign_list_collations? > >>> > >>> I'm asking because in transformSelectStmt, resolveTargetListUnknowns > >>> is invoked before assign_query_collations. It might not matter, but keeping > >>> the order consistent would be good for readers. > >> > >> > >> Updated the patch. It should be before. > > > > Do we really need a test for ORDER BY on a literal column? I replaced > > all the test queries with a single one which covers all the scenarios > > covered by those queries. > > > > The patch needed to consider pstate->p_resolve_unknowns. Unknown > > literals are not resolved as text always. See the test query. > > I couldn't find a commit to apply this patch cleanly (the subject says > patch 5/5, so maybe you had some unpublished local changes?). I am maintaining all the SQL/PGQ fixes in the same branch and creating patches from that branch. Hence 5/5. But still it shouldn't have applied cleanly. Both git cherry-pick and git am <attached patch file>, on a fresh branch succeeded. Please let me know if you still face the issue. > After > applying the test case manually, it looks like the test output is > already correct without the code change. So if this patch is still > required, we need a better test case. > Yes, the test query doesn't reproduce the bug. Actually all but only two queries from the Satya's patch show the bug. I have included a test query which fails, with expected error message, without code changes now. Also I don't think we need a separate section for this test query. Included the query in one of the earliest sections in the file. -- Best Wishes, Ashutosh Bapat Attachments: [text/x-patch] v20260512-0001-Resolve-unknown-type-literals-in-GRAPH_TAB.patch (3.8K, ../../CAExHW5sG6f5m22viW99wgsOmAXXK9312obZJMfZjCUevM7K9rg@mail.gmail.com/2-v20260512-0001-Resolve-unknown-type-literals-in-GRAPH_TAB.patch) download | inline diff: From 69fcbf2b907ff83aaa17593c5596e495cc0720e7 Mon Sep 17 00:00:00 2001 From: Ashutosh Bapat <[email protected]> Date: Wed, 29 Apr 2026 19:07:13 +0530 Subject: [PATCH v20260512 4/4] Resolve unknown-type literals in GRAPH_TABLE COLUMNS The unknown-type literals in the COLUMNS clause of a GRAPH_TABLE are now resolved to the appropriate types, causing various failures. Call resolveTargetListUnknowns() on the columns targetlist to resolve unknonw type literals. Do this before assigning collations so that the resolved types are used for collations assignment. Reported by: Satya Narlapuram <[email protected]> Author: Satya Narlapuram <[email protected]> Author: Ashutosh Bapat <[email protected]> Reviewed by: Junwang Zhao <[email protected]> Discussion: https://www.postgresql.org/message-id/CAHg+QDcyKNWyzDoKMxiZNjv7C-wAxs8y0ZoNkOV137Y+nk3UXg@mail.gmail.com --- src/backend/parser/parse_clause.c | 4 ++++ src/test/regress/expected/graph_table.out | 9 +++++++++ src/test/regress/sql/graph_table.sql | 2 ++ 3 files changed, 15 insertions(+) diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c index 4270c2382c4..3d0585b8efb 100644 --- a/src/backend/parser/parse_clause.c +++ b/src/backend/parser/parse_clause.c @@ -1003,6 +1003,10 @@ transformRangeGraphTable(ParseState *pstate, RangeGraphTable *rgt) columns = lappend(columns, te); } + /* resolve any still-unresolved output columns as being type text */ + if (pstate->p_resolve_unknowns) + resolveTargetListUnknowns(pstate, columns); + /* * Assign collations to column expressions now since * assign_query_collations() does not process rangetable entries. diff --git a/src/test/regress/expected/graph_table.out b/src/test/regress/expected/graph_table.out index cc6d80afd82..e8d49fd5cd4 100644 --- a/src/test/regress/expected/graph_table.out +++ b/src/test/regress/expected/graph_table.out @@ -160,6 +160,15 @@ SELECT * FROM GRAPH_TABLE (myshop MATCH (c IS customers) COLUMNS (c.name)); customer3 (3 rows) +-- Unknown type resolution +SELECT *, pg_typeof(unknown_col) AS unknown_col_type, pg_typeof(null_col) AS null_col_type FROM GRAPH_TABLE (myshop MATCH (c IS customers) COLUMNS (c.name, 'unknown-literal' AS unknown_col, NULL AS null_col)); + name | unknown_col | null_col | unknown_col_type | null_col_type +-----------+-----------------+----------+------------------+--------------- + customer1 | unknown-literal | | text | text + customer2 | unknown-literal | | text | text + customer3 | unknown-literal | | text | text +(3 rows) + SELECT * FROM GRAPH_TABLE (myshop MATCH (c IS customers WHERE c.address = 'US')-[IS customer_orders]->(o IS orders) COLUMNS (c.name)); name ----------- diff --git a/src/test/regress/sql/graph_table.sql b/src/test/regress/sql/graph_table.sql index 0e381ec72bc..e761f09e057 100644 --- a/src/test/regress/sql/graph_table.sql +++ b/src/test/regress/sql/graph_table.sql @@ -134,6 +134,8 @@ INSERT INTO wishlist_items (wishlist_items_id, wishlist_id, product_no) VALUES -- single element path pattern SELECT * FROM GRAPH_TABLE (myshop MATCH (c IS customers) COLUMNS (c.name)); +-- Unknown type resolution +SELECT *, pg_typeof(unknown_col) AS unknown_col_type, pg_typeof(null_col) AS null_col_type FROM GRAPH_TABLE (myshop MATCH (c IS customers) COLUMNS (c.name, 'unknown-literal' AS unknown_col, NULL AS null_col)); SELECT * FROM GRAPH_TABLE (myshop MATCH (c IS customers WHERE c.address = 'US')-[IS customer_orders]->(o IS orders) COLUMNS (c.name)); -- graph element specification without label or variable SELECT * FROM GRAPH_TABLE (myshop MATCH (c IS customers WHERE c.address = 'US')-[]->(o IS orders) COLUMNS (c.name AS customer_name)); -- 2.34.1 ^ permalink raw reply [nested|flat] 60+ messages in thread
* Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS @ 2026-07-03 15:17 Peter Eisentraut <[email protected]> parent: Ashutosh Bapat <[email protected]> 0 siblings, 1 reply; 60+ messages in thread From: Peter Eisentraut @ 2026-07-03 15:17 UTC (permalink / raw) To: Ashutosh Bapat <[email protected]>; +Cc: SATYANARAYANA NARLAPURAM <[email protected]>; Junwang Zhao <[email protected]>; PostgreSQL Hackers <[email protected]> On 12.05.26 08:20, Ashutosh Bapat wrote: > Yes, the test query doesn't reproduce the bug. Actually all but only > two queries from the Satya's patch show the bug. I have included a > test query which fails, with expected error message, without code > changes now. Also I don't think we need a separate section for this > test query. Included the query in one of the earliest sections in the > file. committed ^ permalink raw reply [nested|flat] 60+ messages in thread
* Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS @ 2026-07-06 11:38 Ashutosh Bapat <[email protected]> parent: Peter Eisentraut <[email protected]> 0 siblings, 0 replies; 60+ messages in thread From: Ashutosh Bapat @ 2026-07-06 11:38 UTC (permalink / raw) To: Peter Eisentraut <[email protected]>; +Cc: SATYANARAYANA NARLAPURAM <[email protected]>; Junwang Zhao <[email protected]>; PostgreSQL Hackers <[email protected]> On Fri, Jul 3, 2026 at 8:47 PM Peter Eisentraut <[email protected]> wrote: > > On 12.05.26 08:20, Ashutosh Bapat wrote: > > Yes, the test query doesn't reproduce the bug. Actually all but only > > two queries from the Satya's patch show the bug. I have included a > > test query which fails, with expected error message, without code > > changes now. Also I don't think we need a separate section for this > > test query. Included the query in one of the earliest sections in the > > file. > > committed > Thanks. I actually had another fix in the related area for an issue reported in [1]. I shared the patch on that thread, but missed to do it before. Sorry. Will share rebased patch on that thread. [1] Discussion: https://www.postgresql.org/message-id/[email protected] -- Best Wishes, Ashutosh Bapat ^ permalink raw reply [nested|flat] 60+ messages in thread
end of thread, other threads:[~2026-07-06 11:38 UTC | newest] Thread overview: 60+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v11 7/9] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v18 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v19 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v12 09/11] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v22 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v14 6/8] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v21 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v9 07/11] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v23 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v10 7/9] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v13 6/8] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v20 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2026-04-25 19:52 [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS SATYANARAYANA NARLAPURAM <[email protected]> 2026-04-26 15:10 ` Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS Junwang Zhao <[email protected]> 2026-04-27 06:04 ` Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS SATYANARAYANA NARLAPURAM <[email protected]> 2026-04-29 14:09 ` Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS Ashutosh Bapat <[email protected]> 2026-05-04 15:12 ` Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS Peter Eisentraut <[email protected]> 2026-05-12 06:20 ` Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS Ashutosh Bapat <[email protected]> 2026-07-03 15:17 ` Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS Peter Eisentraut <[email protected]> 2026-07-06 11:38 ` Re: [PATCH] Resolve unknown-type literals in GRAPH_TABLE COLUMNS Ashutosh Bapat <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox