public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v14 6/8] Add pg_ls_dir_recurse to show dir recursively.. 53+ messages / 2 participants [nested] [flat]
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v14 6/8] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? XXX: this test case is unstable, if backends/autovacuum remove FSM... ERROR: could not stat file "./base/16384/20345_fsm": No such file or directory CONTEXT: SQL function "pg_ls_dir_recurse" statement 1 Need catversion bumped ? --- doc/src/sgml/func.sgml | 25 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 +++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 52 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4cdd03610a..b30ac9f10d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21351,6 +21351,16 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21462,6 +21472,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +lateral format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +lateral pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 05a644a7c9..e7295d8aaf 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1436,6 +1436,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 2c5829bca4..af2c2dd621 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10753,6 +10753,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Kynn+LdAwU9N+JqL Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v14-0007-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v22 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/bin/pg_rewind/libpq_fetch.c | 22 +++----------- src/bin/pg_rewind/t/RewindTest.pm | 7 ++++- src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 13 ++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 7 files changed, 69 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index f7ccf68ff4..7baf2b31e5 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25769,6 +25769,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 169602c1b6..55b700dadc 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1510,6 +1510,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index bf4dfc23b9..a9535d3dd5 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -176,30 +176,18 @@ libpqProcessFileList(void) /* * Create a recursive directory listing of the whole data directory. * - * The WITH RECURSIVE part does most of the work. The second part gets the - * targets of the symlinks in pg_tblspc directory. + * Join to pg_tablespace to get the targets of the symlinks in + * pg_tblspc directory. * * XXX: There is no backend function to get a symbolic link's target in * general, so if the admin has put any custom symbolic links in the data * directory, they won't be copied correctly. */ sql = - "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" - " SELECT '' AS path, filename, size, isdir FROM\n" - " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n" - " pg_stat_file(fn.filename, true) AS this\n" - " UNION ALL\n" - " SELECT parent.path || parent.filename || '/' AS path,\n" - " fn, this.size, this.isdir\n" - " FROM files AS parent,\n" - " pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n" - " pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n" - " WHERE parent.isdir = 't'\n" - ")\n" - "SELECT path || filename, size, isdir,\n" + "SELECT COALESCE(NULLIF(path,'.')||'/','')||filename, size, isdir,\n" " pg_tablespace_location(pg_tablespace.oid) AS link_target\n" - "FROM files\n" - "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" + "FROM pg_ls_dir_recurse('.') files\n" + "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc'\n" " AND oid::text = files.filename\n"; res = PQexec(conn, sql); diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm index 41ed7d4b3b..cacd58651c 100644 --- a/src/bin/pg_rewind/t/RewindTest.pm +++ b/src/bin/pg_rewind/t/RewindTest.pm @@ -160,7 +160,12 @@ sub start_primary GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO rewind_user; GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) - TO rewind_user;"); + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_metadata(text, bool, bool) + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_recurse(text) + TO rewind_user; + "); #### Now run the test-specific parts to initialize the primary before setting # up standby diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index c3564168e5..0235042e26 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10976,6 +10976,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,filename,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,text,int8,timestamptz,bool}', + proargnames => '{dirname,path,filename,size,modification,isdir}', proargmodes => '{i,o,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select dirname as path, * from pg_ls_dir_metadata(dirname, false, false) union all select coalesce(nullif(parent.path,'.')||'/','')||parent.filename, a.filename, a.size, a.modification, a.isdir from ls as parent, lateral pg_ls_dir_metadata(parent.path||'/'||parent.filename, false, false) as a where parent.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index bb926a2cf4..302f348a6d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,19 @@ select * from pg_ls_dir_metadata('.') limit 0; ----------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + path | filename | isdir +--------+----------------+------- + pg_wal | archive_status | t +(1 row) + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + path | filename | size | modification | isdir +------+----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 68e2bc6586..fda95828d7 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --d6Gm4EdcadzBjdND Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v22-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v23 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/bin/pg_rewind/libpq_source.c | 22 +++----------- src/bin/pg_rewind/t/RewindTest.pm | 7 ++++- src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 13 ++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 7 files changed, 69 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 729d201249..c95dab6e1d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25760,6 +25760,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index e7647787cf..148dd7a8a8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1504,6 +1504,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/bin/pg_rewind/libpq_source.c b/src/bin/pg_rewind/libpq_source.c index c73e8bf470..46c8b7fcd5 100644 --- a/src/bin/pg_rewind/libpq_source.c +++ b/src/bin/pg_rewind/libpq_source.c @@ -205,30 +205,18 @@ libpq_traverse_files(rewind_source *source, process_file_callback_t callback) /* * Create a recursive directory listing of the whole data directory. * - * The WITH RECURSIVE part does most of the work. The second part gets the - * targets of the symlinks in pg_tblspc directory. + * Join to pg_tablespace to get the targets of the symlinks in + * pg_tblspc directory. * * XXX: There is no backend function to get a symbolic link's target in * general, so if the admin has put any custom symbolic links in the data * directory, they won't be copied correctly. */ sql = - "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" - " SELECT '' AS path, filename, size, isdir FROM\n" - " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n" - " pg_stat_file(fn.filename, true) AS this\n" - " UNION ALL\n" - " SELECT parent.path || parent.filename || '/' AS path,\n" - " fn, this.size, this.isdir\n" - " FROM files AS parent,\n" - " pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n" - " pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n" - " WHERE parent.isdir = 't'\n" - ")\n" - "SELECT path || filename, size, isdir,\n" + "SELECT COALESCE(NULLIF(path,'.')||'/','')||filename, size, isdir,\n" " pg_tablespace_location(pg_tablespace.oid) AS link_target\n" - "FROM files\n" - "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" + "FROM pg_ls_dir_recurse('.') files\n" + "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc'\n" " AND oid::text = files.filename\n"; res = PQexec(conn, sql); diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm index 41ed7d4b3b..cacd58651c 100644 --- a/src/bin/pg_rewind/t/RewindTest.pm +++ b/src/bin/pg_rewind/t/RewindTest.pm @@ -160,7 +160,12 @@ sub start_primary GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO rewind_user; GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) - TO rewind_user;"); + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_metadata(text, bool, bool) + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_recurse(text) + TO rewind_user; + "); #### Now run the test-specific parts to initialize the primary before setting # up standby diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 344d6305e5..5e8f16d5b1 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10983,6 +10983,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,filename,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,text,int8,timestamptz,bool}', + proargnames => '{dirname,path,filename,size,modification,isdir}', proargmodes => '{i,o,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select dirname as path, * from pg_ls_dir_metadata(dirname, false, false) union all select coalesce(nullif(parent.path,'.')||'/','')||parent.filename, a.filename, a.size, a.modification, a.isdir from ls as parent, lateral pg_ls_dir_metadata(parent.path||'/'||parent.filename, false, false) as a where parent.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index bb926a2cf4..302f348a6d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,19 @@ select * from pg_ls_dir_metadata('.') limit 0; ----------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + path | filename | isdir +--------+----------------+------- + pg_wal | archive_status | t +(1 row) + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + path | filename | size | modification | isdir +------+----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 68e2bc6586..fda95828d7 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --mhjHhnbe5PrRcwjY Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v23-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v13 6/8] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? XXX: this test case is unstable, if backends/autovacuum remove FSM... ERROR: could not stat file "./base/16384/20345_fsm": No such file or directory CONTEXT: SQL function "pg_ls_dir_recurse" statement 1 Need catversion bumped ? --- doc/src/sgml/func.sgml | 25 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 +++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 52 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4cdd03610a..0763db3f50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21351,6 +21351,16 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21462,6 +21472,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) AS suffix) AS dir, +pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 05a644a7c9..e7295d8aaf 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1436,6 +1436,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index be3e820a03..b3509f0e12 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10747,6 +10747,12 @@ proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 3ca4b85dd9..ae8716a83f 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -208,6 +208,20 @@ select * from pg_ls_tmpdir() where name='Does not exist'; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index f09c890b5d..7038bcdb0f 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -64,6 +64,12 @@ select count(*) > 0 from -- The name='' condition is never true, so the function runs to completion but returns zero rows. select * from pg_ls_tmpdir() where name='Does not exist'; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --E0h0CbphJD8hN+Gf Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0007-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v9 07/11] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? TODO: src/backend/catalog/system_views.sql:REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; Need catversion bumped ? --- doc/src/sgml/func.sgml | 13 +++++++++++++ src/include/catalog/pg_proc.dat | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7c73d3b82a..672cbab7b9 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21442,6 +21442,19 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + <parameter>missing_ok</parameter> ??? + To recusively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp')dir FROM pg_tablespace b, pg_control_system() pcs, format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) suffix)dir, pg_ls_dir_recurse(dir)a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7789d029ea..cc2c6f6571 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -6143,6 +6143,14 @@ proname => 'pg_ls_dir', prorows => '1000', proretset => 't', provolatile => 'v', prorettype => 'text', proargtypes => 'text bool bool', prosrc => 'pg_ls_dir' }, + +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "WITH RECURSIVE x AS (SELECT * FROM pg_ls_dir_metadata(dirname, true, false, true) UNION ALL SELECT x.name||'/'||a.name, a.size, a.modification, a.isdir FROM x, pg_ls_dir_metadata(dirname||'/'||x.name, true, false, true)a WHERE x.isdir) SELECT * FROM x" }, + { oid => '2626', descr => 'sleep for the specified time in seconds', proname => 'pg_sleep', provolatile => 'v', prorettype => 'void', proargtypes => 'float8', prosrc => 'pg_sleep' }, -- 2.17.0 --32u276st3Jlj2kUU Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0008-generalize-pg_ls_dir_files-and-retire-pg_ls_dir.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v19 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d1d2f868d8..c097977d9b 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25733,6 +25733,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index f5217a0d93..297b0dcdb8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1482,6 +1482,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b61f30247e..bac1226db7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10914,6 +10914,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false) as a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --S0GG+JvAI2G0KxBG Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v19-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v18 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4aa9cdb18c..683f533392 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index cc8e870740..58d535a9f6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1469,6 +1469,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b61f30247e..bac1226db7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10914,6 +10914,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false) as a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --5I6of5zJg18YgZEa Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v18-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v10 7/9] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? TODO: src/backend/catalog/system_views.sql:REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; Need catversion bumped ? --- doc/src/sgml/func.sgml | 13 +++++++++++++ src/include/catalog/pg_proc.dat | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 3d8f7dff96..d0b782d803 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21445,6 +21445,19 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + <parameter>missing_ok</parameter> ??? + To recusively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp')dir FROM pg_tablespace b, pg_control_system() pcs, format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) suffix)dir, pg_ls_dir_recurse(dir)a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b4c3d15495..75423db22e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -6144,6 +6144,13 @@ provolatile => 'v', prorettype => 'text', proargtypes => 'text bool bool', prosrc => 'pg_ls_dir' }, +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "WITH RECURSIVE x AS (SELECT * FROM pg_ls_dir_metadata(dirname, true, false, true) UNION ALL SELECT x.name||'/'||a.name, a.size, a.modification, a.isdir FROM x, pg_ls_dir_metadata(dirname||'/'||x.name, true, false, true)a WHERE x.isdir) SELECT * FROM x" }, + { oid => '2626', descr => 'sleep for the specified time in seconds', proname => 'pg_sleep', provolatile => 'v', prorettype => 'void', proargtypes => 'float8', prosrc => 'pg_sleep' }, -- 2.17.0 --rCwQ2Y43eQY6RBgR Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v12 09/11] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? TODO: src/backend/catalog/system_views.sql:REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; Need catversion bumped ? --- doc/src/sgml/func.sgml | 24 ++++++++++++++++++++ src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 4 files changed, 50 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4cdd03610a..373e3b647b 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21351,6 +21351,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified directory, along with each file's metadata. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21462,6 +21471,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) AS suffix) AS dir, +pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b4c3d15495..667ea5721e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10748,6 +10748,12 @@ proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 448901d841..82d967a62d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -187,6 +187,20 @@ select * from pg_ls_tmpdir() where name='Does not exist'; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 656aad93e6..2cfdac386d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -55,6 +55,12 @@ select count(*) >= 0 as ok from pg_ls_archive_statusdir(); -- The name='' condition is never true, so the function runs to completion but returns zero rows. select * from pg_ls_tmpdir() where name='Does not exist'; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --wIc/V6YLA2QdyfT4 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0010-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v20 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 13 ++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 58 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9652409581..befd2ffca5 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25741,6 +25741,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 360b6bda26..44690be916 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1479,6 +1479,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 0b5716525c..e60b6ca866 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10931,6 +10931,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,text,int8,timestamptz,bool}', + proargnames => '{dirname,path,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select dirname as path, * from pg_ls_dir_metadata(dirname, true, false) union all select parent.path||'/'||parent.name, a.name, a.size, a.modification, a.isdir from ls AS parent, lateral pg_ls_dir_metadata(parent.path||'/'||parent.name, false, false) as a where parent.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..07c0a7f961 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,19 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT path, name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND path='./pg_wal'; + path | name | isdir +----------+----------------+------- + ./pg_wal | archive_status | t +(1 row) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + path | name | size | modification | isdir +------+------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..349752da94 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT path, name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND path='./pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Z1Z8UV8BNhgCynIS Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v20-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v21 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/bin/pg_rewind/libpq_fetch.c | 22 +++----------- src/bin/pg_rewind/t/RewindTest.pm | 7 ++++- src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 13 ++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 7 files changed, 69 insertions(+), 18 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9652409581..befd2ffca5 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25741,6 +25741,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b9b2a6aa20..cdf105ffac 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1479,6 +1479,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index c44648f823..58f2bb9fd1 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -176,30 +176,18 @@ libpqProcessFileList(void) /* * Create a recursive directory listing of the whole data directory. * - * The WITH RECURSIVE part does most of the work. The second part gets the - * targets of the symlinks in pg_tblspc directory. + * Join to pg_tablespace to get the targets of the symlinks in + * pg_tblspc directory. * * XXX: There is no backend function to get a symbolic link's target in * general, so if the admin has put any custom symbolic links in the data * directory, they won't be copied correctly. */ sql = - "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" - " SELECT '' AS path, filename, size, isdir FROM\n" - " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n" - " pg_stat_file(fn.filename, true) AS this\n" - " UNION ALL\n" - " SELECT parent.path || parent.filename || '/' AS path,\n" - " fn, this.size, this.isdir\n" - " FROM files AS parent,\n" - " pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n" - " pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n" - " WHERE parent.isdir = 't'\n" - ")\n" - "SELECT path || filename, size, isdir,\n" + "SELECT COALESCE(NULLIF(path,'.')||'/','')||filename, size, isdir,\n" " pg_tablespace_location(pg_tablespace.oid) AS link_target\n" - "FROM files\n" - "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" + "FROM pg_ls_dir_recurse('.') files\n" + "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc'\n" " AND oid::text = files.filename\n"; res = PQexec(conn, sql); diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm index 7516af7a01..bc8dad074e 100644 --- a/src/bin/pg_rewind/t/RewindTest.pm +++ b/src/bin/pg_rewind/t/RewindTest.pm @@ -160,7 +160,12 @@ sub start_primary GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO rewind_user; GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) - TO rewind_user;"); + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_metadata(text, bool, bool) + TO rewind_user; + GRANT EXECUTE ON function pg_catalog.pg_ls_dir_recurse(text) + TO rewind_user; + "); #### Now run the test-specific parts to initialize the primary before setting # up standby diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 58d1c74b52..826a31c561 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10931,6 +10931,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,filename,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '9981', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,text,int8,timestamptz,bool}', + proargnames => '{dirname,path,filename,size,modification,isdir}', proargmodes => '{i,o,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select dirname as path, * from pg_ls_dir_metadata(dirname, false, false) union all select coalesce(nullif(parent.path,'.')||'/','')||parent.filename, a.filename, a.size, a.modification, a.isdir from ls as parent, lateral pg_ls_dir_metadata(parent.path||'/'||parent.filename, false, false) as a where parent.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index bb926a2cf4..302f348a6d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,19 @@ select * from pg_ls_dir_metadata('.') limit 0; ----------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + path | filename | isdir +--------+----------------+------- + pg_wal | archive_status | t +(1 row) + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + path | filename | size | modification | isdir +------+----------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 68e2bc6586..fda95828d7 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select filename, isdir from pg_ls_dir_metadata('.', false, false) where filename -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +select path, filename, isdir from pg_ls_dir_recurse('.') where isdir and path='pg_wal'; + +-- Check that expected columns are present +select * from pg_ls_dir_recurse('.') limit 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --Tcb1KvpfnM4LxW2s Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v21-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v11 7/9] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? TODO: src/backend/catalog/system_views.sql:REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; Need catversion bumped ? --- doc/src/sgml/func.sgml | 24 ++++++++++++++++++++ src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 4 files changed, 50 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index e4ba3fee40..f3422808d9 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21351,6 +21351,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); For each file in a directory, list the file and its metadata. Restricted to superusers by default, but other users can be granted EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified directory, along with each file's metadata. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -21462,6 +21471,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +format('/PG_%s_%s', left(current_setting('server_version_num'), 2), catalog_version_no) AS suffix) AS dir, +pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index b4c3d15495..667ea5721e 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10748,6 +10748,12 @@ proallargtypes => '{text,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,o,o,o,o}', proargnames => '{dirname,missing_ok,include_dot_dirs,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata' }, +{ oid => '8511', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 448901d841..82d967a62d 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -187,6 +187,20 @@ select * from pg_ls_tmpdir() where name='Does not exist'; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 656aad93e6..2cfdac386d 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -55,6 +55,12 @@ select count(*) >= 0 as ok from pg_ls_archive_statusdir(); -- The name='' condition is never true, so the function runs to completion but returns zero rows. select * from pg_ls_tmpdir() where name='Does not exist'; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --oTHb8nViIGeoXxdp Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v11-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 32 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 ++++ src/test/regress/expected/misc_functions.out | 14 +++++++++ src/test/regress/sql/misc_functions.sql | 6 ++++ 5 files changed, 59 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8e0de7c02d..334b6beb3a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25738,6 +25738,38 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <function>pg_ls_dir_recurse</function> ( <parameter>dirname</parameter> <type>text</type> ) + <returnvalue>setof record</returnvalue> + ( <parameter>name</parameter> <type>text</type>, + <parameter>size</parameter> <type>bigint</type>, + <parameter>modification</parameter> <type>timestamp with time zone</type>, + <parameter>isdir</parameter> <type>boolean</type> ) + </para> + <para> + Recursively list each file in the specified directory, along with the + files' metadata. + </para> + <para> + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </para></entry> + </row> + + <!--para> + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para--> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --4LFBTxd4L5NLO6ly Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. @ 2020-03-09 03:52 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Justin Pryzby @ 2020-03-09 03:52 UTC (permalink / raw) ..possibly there's a better place to put this, like maybe a doc-only example ? Need catversion bumped ? --- doc/src/sgml/func.sgml | 27 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 1 + src/include/catalog/pg_proc.dat | 6 +++++ src/test/regress/expected/misc_functions.out | 14 ++++++++++ src/test/regress/sql/misc_functions.sql | 6 +++++ 5 files changed, 54 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index c3a6b3277f..3a2ad3eb50 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25311,6 +25311,18 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); EXECUTE to run the function. </entry> </row> + <row> + <entry> + <literal><function>pg_ls_dir_recurse(<parameter>dirname</parameter> <type>text</type>)</function></literal> + </entry> + <entry><type>setof text</type></entry> + <entry> + Call pg_ls_dir_metadata to recursively list the files in the specified + directory, along with each file's metadata. + Restricted to superusers by default, but other users can be granted + EXECUTE to run the function. + </entry> + </row> <row> <entry> <literal><function>pg_ls_logdir()</function></literal> @@ -25422,6 +25434,21 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); directory along with the file's metadata. </para> + <indexterm> + <primary>pg_ls_dir_recurse</primary> + </indexterm> + <para> + <function>pg_ls_dir_recurse</function> recursively lists the files + in the specified directory. + To recursively list temporary directories in all tablespaces: +<programlisting> +SELECT * FROM (SELECT DISTINCT COALESCE(NULLIF(pg_tablespace_location(b.oid),'')||suffix, 'base/pgsql_tmp') AS dir +FROM pg_tablespace b, pg_control_system() pcs, +LATERAL format('/PG_%s_%s', left(current_setting('server_version_num'), 2), pcs.catalog_version_no) AS suffix) AS dir, +LATERAL pg_ls_dir_recurse(dir) AS a; +</programlisting> + </para> + <indexterm> <primary>pg_ls_logdir</primary> </indexterm> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 1c77430f0c..215fba3b7d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1468,6 +1468,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_dir_recurse(text) FROM public; -- -- We also set up some things as accessible to standard roles. diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index adfce45d1a..f94f403475 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10916,6 +10916,12 @@ proallargtypes => '{text,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}', proargnames => '{dirname,name,size,modification,isdir}', prosrc => 'pg_ls_dir_metadata_1arg' }, +{ oid => '5034', descr => 'list all files in a directory recursively', + proname => 'pg_ls_dir_recurse', prorows => '10000', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => 'text', + proallargtypes => '{text,text,int8,timestamptz,bool}', + proargnames => '{dirname,name,size,modification,isdir}', proargmodes => '{i,o,o,o,o}', + prolang => 'sql', prosrc => "with recursive ls as (select * from pg_ls_dir_metadata(dirname, true, false) union all select ls.name||'/'||a.name, a.size, a.modification, a.isdir from ls, lateral pg_ls_dir_metadata(dirname||'/'||ls.name, false, false)a where ls.isdir) select * from ls" }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 64b1417fb8..4188d684f0 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -243,6 +243,20 @@ select * from pg_ls_dir_metadata('.') limit 0; ------+------+--------------+------- (0 rows) +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + name | isdir +-----------------------+------- + pg_wal | t + pg_wal/archive_status | t +(2 rows) + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + name | size | modification | isdir +------+------+--------------+------- +(0 rows) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 372345720d..6041c4f3dc 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -76,6 +76,12 @@ select name, isdir from pg_ls_dir_metadata('.', false, false) where name='.'; -- -- Check that expected columns are present select * from pg_ls_dir_metadata('.') limit 0; +-- Check that we at least succeed in recursing once, and that we don't show the leading dir prefix +SELECT name, isdir FROM pg_ls_dir_recurse('.') WHERE isdir AND name~'^pg_wal'; + +-- Check that expected columns are present +SELECT * FROM pg_ls_dir_recurse('.') LIMIT 0; + -- -- Test adding a support function to a subject function -- -- 2.17.0 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0008-pg_ls_logdir-to-ignore-error-if-initial-top-dir-.patch" ^ permalink raw reply [nested|flat] 53+ messages in thread
* Remove redundant NULL check in clause_selectivity_ext() function @ 2024-08-19 09:38 Ilia Evdokimov <[email protected]> 0 siblings, 0 replies; 53+ messages in thread From: Ilia Evdokimov @ 2024-08-19 09:38 UTC (permalink / raw) To: pgsql-hackers Hi everyone, In the `clause_selectivity_ext()` function, there’s a comment regarding a NULL clause check that asks, "can this still happen?". I decided to investigate whether this scenario is still valid. Here’s what I found after reviewing the relevant cases: - approx_tuple_count(): The function iterates over a loop of other clauses. - get_foreign_key_join_selectivity(): The function is invoked within an `if (clause)` condition. - consider_new_or_clause(): The clause is generated by `make_restrictinfo()`, which never returns NULL. - clauselist_selectivity_ext(): This function either checks `list_length(clauses) == 1` before being called, or it is called within a loop of clauses. In other cases, the function is called recursively from `clause_selectivity_ext()`. If you are aware of any situations where a NULL clause could be passed or if I’ve missed anything, please let me know. I’m also open to any other suggestions. -- Regards, Ilia Evdokimov, Tantor Labs LCC. Attachments: [text/x-patch] v1-0001-Remove-redundant-NULL-check-for-clause-during-select.patch (1.1K, ../../[email protected]/2-v1-0001-Remove-redundant-NULL-check-for-clause-during-select.patch) download | inline diff: From fab0575ef1350cb700b6fc079230397ecb5ca19d Mon Sep 17 00:00:00 2001 From: Ilia Evdokimov <[email protected]> Date: Mon, 19 Aug 2024 12:08:53 +0300 Subject: [PATCH] Remove redundant NULL check for clause during selectivity calculation The function clause_selectivity_ext() takes a Node *clause parameter, which is either iterated over in a loop with other clauses or passed as a RestrictInfo from other functions. Since these functions guarantee that clause cannot be NULL, the NULL check is unnecessary and has been removed. --- src/backend/optimizer/path/clausesel.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/backend/optimizer/path/clausesel.c b/src/backend/optimizer/path/clausesel.c index 0ab021c1e8..5992f96ed2 100644 --- a/src/backend/optimizer/path/clausesel.c +++ b/src/backend/optimizer/path/clausesel.c @@ -692,9 +692,6 @@ clause_selectivity_ext(PlannerInfo *root, RestrictInfo *rinfo = NULL; bool cacheable = false; - if (clause == NULL) /* can this still happen? */ - return s1; - if (IsA(clause, RestrictInfo)) { rinfo = (RestrictInfo *) clause; -- 2.34.1 ^ permalink raw reply [nested|flat] 53+ messages in thread
end of thread, other threads:[~2024-08-19 09:38 UTC | newest] Thread overview: 53+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v14 6/8] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v22 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v23 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v13 6/8] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v9 07/11] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v19 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v18 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v10 7/9] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v12 09/11] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v20 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v21 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v11 7/9] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v17 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2020-03-09 03:52 [PATCH v16 07/10] Add pg_ls_dir_recurse to show dir recursively.. Justin Pryzby <[email protected]> 2024-08-19 09:38 Remove redundant NULL check in clause_selectivity_ext() function Ilia Evdokimov <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox