agora inbox for [email protected]
help / color / mirror / Atom feed[PATCH 1/8] Make syntax changes for custom compression methods
6+ messages / 5 participants
[nested] [flat]
* [PATCH 1/8] Make syntax changes for custom compression methods
@ 2018-06-18 09:25 Ildus Kurbangaliev <[email protected]>
0 siblings, 0 replies; 6+ messages in thread
From: Ildus Kurbangaliev @ 2018-06-18 09:25 UTC (permalink / raw)
---
src/backend/parser/gram.y | 76 ++++++++++++++++++++++++++++++----
src/include/catalog/pg_am.h | 1 +
src/include/nodes/nodes.h | 1 +
src/include/nodes/parsenodes.h | 19 ++++++++-
src/include/parser/kwlist.h | 1 +
5 files changed, 89 insertions(+), 9 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 90dfac2cb1..ab8fa0b7dd 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -311,6 +311,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
opt_grant_grant_option opt_grant_admin_option
opt_nowait opt_if_exists opt_with_data
%type <ival> opt_nowait_or_skip
+%type <ival> AccessMethodType
%type <list> OptRoleList AlterOptRoleList
%type <defelt> CreateOptRoleElem AlterOptRoleElem
@@ -399,6 +400,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
transform_element_list transform_type_list
TriggerTransitions TriggerReferencing
publication_name_list
+ optCompressionParameters
vacuum_relation_list opt_vacuum_relation_list
%type <list> group_by_list
@@ -585,6 +587,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> hash_partbound partbound_datum_list range_datum_list
%type <defelt> hash_partbound_elem
+%type <node> optColumnCompression alterColumnCompression
+%type <str> compressionClause
+%type <list> optCompressionPreserve
+
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -617,9 +623,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P
CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT
- COMMITTED CONCURRENTLY CONFIGURATION CONFLICT CONNECTION CONSTRAINT
- CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE
- CROSS CSV CUBE CURRENT_P
+ COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT
+ CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY
+ COST CREATE CROSS CSV CUBE CURRENT_P
CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
@@ -2214,6 +2220,15 @@ alter_table_cmd:
n->missing_ok = true;
$$ = (Node *)n;
}
+ /* ALTER TABLE <name> ALTER [COLUMN] <colname> SET (COMPRESSION <cm> [WITH (<options>)]) */
+ | ALTER opt_column ColId SET alterColumnCompression
+ {
+ AlterTableCmd *n = makeNode(AlterTableCmd);
+ n->subtype = AT_SetCompression;
+ n->name = $3;
+ n->def = $5;
+ $$ = (Node *)n;
+ }
/* ALTER TABLE <name> DROP [COLUMN] IF EXISTS <colname> [RESTRICT|CASCADE] */
| DROP opt_column IF_P EXISTS ColId opt_drop_behavior
{
@@ -3380,11 +3395,12 @@ TypedTableElement:
| TableConstraint { $$ = $1; }
;
-columnDef: ColId Typename create_generic_options ColQualList
+columnDef: ColId Typename optColumnCompression create_generic_options ColQualList
{
ColumnDef *n = makeNode(ColumnDef);
n->colname = $1;
n->typeName = $2;
+ n->compression = (ColumnCompression *) $3;
n->inhcount = 0;
n->is_local = true;
n->is_not_null = false;
@@ -3394,8 +3410,8 @@ columnDef: ColId Typename create_generic_options ColQualList
n->raw_default = NULL;
n->cooked_default = NULL;
n->collOid = InvalidOid;
- n->fdwoptions = $3;
- SplitColQualList($4, &n->constraints, &n->collClause,
+ n->fdwoptions = $4;
+ SplitColQualList($5, &n->constraints, &n->collClause,
yyscanner);
n->location = @1;
$$ = (Node *)n;
@@ -3442,6 +3458,43 @@ columnOptions: ColId ColQualList
}
;
+optCompressionPreserve:
+ PRESERVE '(' name_list ')' { $$ = $3; }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+optCompressionParameters:
+ WITH '(' generic_option_list ')' { $$ = $3; }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+compressionClause:
+ COMPRESSION name { $$ = pstrdup($2); }
+ ;
+
+optColumnCompression:
+ compressionClause optCompressionParameters
+ {
+ ColumnCompression *n = makeNode(ColumnCompression);
+ n->amname = $1;
+ n->options = (List *) $2;
+ n->preserve = NIL;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+alterColumnCompression:
+ compressionClause optCompressionParameters optCompressionPreserve
+ {
+ ColumnCompression *n = makeNode(ColumnCompression);
+ n->amname = $1;
+ n->options = (List *) $2;
+ n->preserve = (List *) $3;
+ $$ = (Node *) n;
+ }
+ ;
+
ColQualList:
ColQualList ColConstraint { $$ = lappend($1, $2); }
| /*EMPTY*/ { $$ = NIL; }
@@ -3647,6 +3700,7 @@ TableLikeOption:
| INDEXES { $$ = CREATE_TABLE_LIKE_INDEXES; }
| STATISTICS { $$ = CREATE_TABLE_LIKE_STATISTICS; }
| STORAGE { $$ = CREATE_TABLE_LIKE_STORAGE; }
+ | COMPRESSION { $$ = CREATE_TABLE_LIKE_COMPRESSION; }
| ALL { $$ = CREATE_TABLE_LIKE_ALL; }
;
@@ -5324,12 +5378,17 @@ row_security_cmd:
*
*****************************************************************************/
-CreateAmStmt: CREATE ACCESS METHOD name TYPE_P INDEX HANDLER handler_name
+AccessMethodType:
+ INDEX { $$ = AMTYPE_INDEX; }
+ | COMPRESSION { $$ = AMTYPE_COMPRESSION; }
+ ;
+
+CreateAmStmt: CREATE ACCESS METHOD name TYPE_P AccessMethodType HANDLER handler_name
{
CreateAmStmt *n = makeNode(CreateAmStmt);
n->amname = $4;
n->handler_name = $8;
- n->amtype = AMTYPE_INDEX;
+ n->amtype = $6;
$$ = (Node *) n;
}
;
@@ -15031,6 +15090,7 @@ unreserved_keyword:
| COMMENTS
| COMMIT
| COMMITTED
+ | COMPRESSION
| CONFIGURATION
| CONFLICT
| CONNECTION
diff --git a/src/include/catalog/pg_am.h b/src/include/catalog/pg_am.h
index 26cb234987..a3b75c5256 100644
--- a/src/include/catalog/pg_am.h
+++ b/src/include/catalog/pg_am.h
@@ -51,6 +51,7 @@ typedef FormData_pg_am *Form_pg_am;
* Allowed values for amtype
*/
#define AMTYPE_INDEX 'i' /* index access method */
+#define AMTYPE_COMPRESSION 'c' /* compression access method */
#endif /* EXPOSE_TO_CLIENT_CODE */
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 43f1552241..03cee8ec88 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -473,6 +473,7 @@ typedef enum NodeTag
T_PartitionBoundSpec,
T_PartitionRangeDatum,
T_PartitionCmd,
+ T_ColumnCompression,
T_VacuumRelation,
/*
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 6390f7e8c1..69a03d4388 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -623,6 +623,20 @@ typedef struct RangeTableSample
int location; /* method name location, or -1 if unknown */
} RangeTableSample;
+/*
+ * ColumnCompression - compression parameters for some attribute
+ *
+ * This represents compression information defined using clause:
+ * .. COMPRESSION <compression method> WITH (<params>) PRESERVE <compression AMs>
+ */
+typedef struct ColumnCompression
+{
+ NodeTag type;
+ char *amname;
+ List *options;
+ List *preserve;
+} ColumnCompression;
+
/*
* ColumnDef - column definition (used in various creates)
*
@@ -646,6 +660,7 @@ typedef struct ColumnDef
NodeTag type;
char *colname; /* name of column */
TypeName *typeName; /* type of column */
+ ColumnCompression *compression;
int inhcount; /* number of times column is inherited */
bool is_local; /* column has local (non-inherited) def'n */
bool is_not_null; /* NOT NULL constraint specified? */
@@ -683,6 +698,7 @@ typedef enum TableLikeOption
CREATE_TABLE_LIKE_INDEXES = 1 << 4,
CREATE_TABLE_LIKE_STATISTICS = 1 << 5,
CREATE_TABLE_LIKE_STORAGE = 1 << 6,
+ CREATE_TABLE_LIKE_COMPRESSION = 1 << 7,
CREATE_TABLE_LIKE_ALL = PG_INT32_MAX
} TableLikeOption;
@@ -1790,7 +1806,8 @@ typedef enum AlterTableType
AT_DetachPartition, /* DETACH PARTITION */
AT_AddIdentity, /* ADD IDENTITY */
AT_SetIdentity, /* SET identity column options */
- AT_DropIdentity /* DROP IDENTITY */
+ AT_DropIdentity, /* DROP IDENTITY */
+ AT_SetCompression /* SET COMPRESSION */
} AlterTableType;
typedef struct ReplicaIdentityStmt
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 23db40147b..76c33cbc5f 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -88,6 +88,7 @@ PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD)
PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD)
PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD)
PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD)
+PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD)
PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD)
PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD)
PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD)
--
2.17.1
--MP_/bPAPXLM_Co3=OWSPDA+8yq.
Content-Type: text/x-patch
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename=0002-Add-compression-catalog-tables-and-the-basic-inf-v16.patch
^ permalink raw reply [nested|flat] 6+ messages in thread
* [PATCH 1/8] Make syntax changes for custom compression methods
@ 2018-06-18 09:25 Ildus Kurbangaliev <[email protected]>
0 siblings, 0 replies; 6+ messages in thread
From: Ildus Kurbangaliev @ 2018-06-18 09:25 UTC (permalink / raw)
---
src/backend/parser/gram.y | 76 ++++++++++++++++++++++++++++++----
src/include/catalog/pg_am.h | 1 +
src/include/nodes/nodes.h | 1 +
src/include/nodes/parsenodes.h | 19 ++++++++-
src/include/parser/kwlist.h | 1 +
5 files changed, 89 insertions(+), 9 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 90dfac2cb1..ab8fa0b7dd 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -311,6 +311,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
opt_grant_grant_option opt_grant_admin_option
opt_nowait opt_if_exists opt_with_data
%type <ival> opt_nowait_or_skip
+%type <ival> AccessMethodType
%type <list> OptRoleList AlterOptRoleList
%type <defelt> CreateOptRoleElem AlterOptRoleElem
@@ -399,6 +400,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
transform_element_list transform_type_list
TriggerTransitions TriggerReferencing
publication_name_list
+ optCompressionParameters
vacuum_relation_list opt_vacuum_relation_list
%type <list> group_by_list
@@ -585,6 +587,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> hash_partbound partbound_datum_list range_datum_list
%type <defelt> hash_partbound_elem
+%type <node> optColumnCompression alterColumnCompression
+%type <str> compressionClause
+%type <list> optCompressionPreserve
+
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -617,9 +623,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P
CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT
- COMMITTED CONCURRENTLY CONFIGURATION CONFLICT CONNECTION CONSTRAINT
- CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE
- CROSS CSV CUBE CURRENT_P
+ COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT
+ CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY
+ COST CREATE CROSS CSV CUBE CURRENT_P
CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
@@ -2214,6 +2220,15 @@ alter_table_cmd:
n->missing_ok = true;
$$ = (Node *)n;
}
+ /* ALTER TABLE <name> ALTER [COLUMN] <colname> SET (COMPRESSION <cm> [WITH (<options>)]) */
+ | ALTER opt_column ColId SET alterColumnCompression
+ {
+ AlterTableCmd *n = makeNode(AlterTableCmd);
+ n->subtype = AT_SetCompression;
+ n->name = $3;
+ n->def = $5;
+ $$ = (Node *)n;
+ }
/* ALTER TABLE <name> DROP [COLUMN] IF EXISTS <colname> [RESTRICT|CASCADE] */
| DROP opt_column IF_P EXISTS ColId opt_drop_behavior
{
@@ -3380,11 +3395,12 @@ TypedTableElement:
| TableConstraint { $$ = $1; }
;
-columnDef: ColId Typename create_generic_options ColQualList
+columnDef: ColId Typename optColumnCompression create_generic_options ColQualList
{
ColumnDef *n = makeNode(ColumnDef);
n->colname = $1;
n->typeName = $2;
+ n->compression = (ColumnCompression *) $3;
n->inhcount = 0;
n->is_local = true;
n->is_not_null = false;
@@ -3394,8 +3410,8 @@ columnDef: ColId Typename create_generic_options ColQualList
n->raw_default = NULL;
n->cooked_default = NULL;
n->collOid = InvalidOid;
- n->fdwoptions = $3;
- SplitColQualList($4, &n->constraints, &n->collClause,
+ n->fdwoptions = $4;
+ SplitColQualList($5, &n->constraints, &n->collClause,
yyscanner);
n->location = @1;
$$ = (Node *)n;
@@ -3442,6 +3458,43 @@ columnOptions: ColId ColQualList
}
;
+optCompressionPreserve:
+ PRESERVE '(' name_list ')' { $$ = $3; }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+optCompressionParameters:
+ WITH '(' generic_option_list ')' { $$ = $3; }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+compressionClause:
+ COMPRESSION name { $$ = pstrdup($2); }
+ ;
+
+optColumnCompression:
+ compressionClause optCompressionParameters
+ {
+ ColumnCompression *n = makeNode(ColumnCompression);
+ n->amname = $1;
+ n->options = (List *) $2;
+ n->preserve = NIL;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+alterColumnCompression:
+ compressionClause optCompressionParameters optCompressionPreserve
+ {
+ ColumnCompression *n = makeNode(ColumnCompression);
+ n->amname = $1;
+ n->options = (List *) $2;
+ n->preserve = (List *) $3;
+ $$ = (Node *) n;
+ }
+ ;
+
ColQualList:
ColQualList ColConstraint { $$ = lappend($1, $2); }
| /*EMPTY*/ { $$ = NIL; }
@@ -3647,6 +3700,7 @@ TableLikeOption:
| INDEXES { $$ = CREATE_TABLE_LIKE_INDEXES; }
| STATISTICS { $$ = CREATE_TABLE_LIKE_STATISTICS; }
| STORAGE { $$ = CREATE_TABLE_LIKE_STORAGE; }
+ | COMPRESSION { $$ = CREATE_TABLE_LIKE_COMPRESSION; }
| ALL { $$ = CREATE_TABLE_LIKE_ALL; }
;
@@ -5324,12 +5378,17 @@ row_security_cmd:
*
*****************************************************************************/
-CreateAmStmt: CREATE ACCESS METHOD name TYPE_P INDEX HANDLER handler_name
+AccessMethodType:
+ INDEX { $$ = AMTYPE_INDEX; }
+ | COMPRESSION { $$ = AMTYPE_COMPRESSION; }
+ ;
+
+CreateAmStmt: CREATE ACCESS METHOD name TYPE_P AccessMethodType HANDLER handler_name
{
CreateAmStmt *n = makeNode(CreateAmStmt);
n->amname = $4;
n->handler_name = $8;
- n->amtype = AMTYPE_INDEX;
+ n->amtype = $6;
$$ = (Node *) n;
}
;
@@ -15031,6 +15090,7 @@ unreserved_keyword:
| COMMENTS
| COMMIT
| COMMITTED
+ | COMPRESSION
| CONFIGURATION
| CONFLICT
| CONNECTION
diff --git a/src/include/catalog/pg_am.h b/src/include/catalog/pg_am.h
index 26cb234987..a3b75c5256 100644
--- a/src/include/catalog/pg_am.h
+++ b/src/include/catalog/pg_am.h
@@ -51,6 +51,7 @@ typedef FormData_pg_am *Form_pg_am;
* Allowed values for amtype
*/
#define AMTYPE_INDEX 'i' /* index access method */
+#define AMTYPE_COMPRESSION 'c' /* compression access method */
#endif /* EXPOSE_TO_CLIENT_CODE */
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 43f1552241..03cee8ec88 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -473,6 +473,7 @@ typedef enum NodeTag
T_PartitionBoundSpec,
T_PartitionRangeDatum,
T_PartitionCmd,
+ T_ColumnCompression,
T_VacuumRelation,
/*
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 6390f7e8c1..69a03d4388 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -623,6 +623,20 @@ typedef struct RangeTableSample
int location; /* method name location, or -1 if unknown */
} RangeTableSample;
+/*
+ * ColumnCompression - compression parameters for some attribute
+ *
+ * This represents compression information defined using clause:
+ * .. COMPRESSION <compression method> WITH (<params>) PRESERVE <compression AMs>
+ */
+typedef struct ColumnCompression
+{
+ NodeTag type;
+ char *amname;
+ List *options;
+ List *preserve;
+} ColumnCompression;
+
/*
* ColumnDef - column definition (used in various creates)
*
@@ -646,6 +660,7 @@ typedef struct ColumnDef
NodeTag type;
char *colname; /* name of column */
TypeName *typeName; /* type of column */
+ ColumnCompression *compression;
int inhcount; /* number of times column is inherited */
bool is_local; /* column has local (non-inherited) def'n */
bool is_not_null; /* NOT NULL constraint specified? */
@@ -683,6 +698,7 @@ typedef enum TableLikeOption
CREATE_TABLE_LIKE_INDEXES = 1 << 4,
CREATE_TABLE_LIKE_STATISTICS = 1 << 5,
CREATE_TABLE_LIKE_STORAGE = 1 << 6,
+ CREATE_TABLE_LIKE_COMPRESSION = 1 << 7,
CREATE_TABLE_LIKE_ALL = PG_INT32_MAX
} TableLikeOption;
@@ -1790,7 +1806,8 @@ typedef enum AlterTableType
AT_DetachPartition, /* DETACH PARTITION */
AT_AddIdentity, /* ADD IDENTITY */
AT_SetIdentity, /* SET identity column options */
- AT_DropIdentity /* DROP IDENTITY */
+ AT_DropIdentity, /* DROP IDENTITY */
+ AT_SetCompression /* SET COMPRESSION */
} AlterTableType;
typedef struct ReplicaIdentityStmt
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 23db40147b..76c33cbc5f 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -88,6 +88,7 @@ PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD)
PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD)
PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD)
PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD)
+PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD)
PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD)
PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD)
PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD)
--
2.18.0
--MP_/w9zXrmKKcGlmI.IAcoy33yI
Content-Type: text/x-patch
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename=0002-Add-compression-catalog-tables-and-the-basic-inf-v17.patch
^ permalink raw reply [nested|flat] 6+ messages in thread
* [PATCH v11 4/4] Various tweaks, mostly naming
@ 2020-02-11 23:12 Alvaro Herrera <[email protected]>
0 siblings, 0 replies; 6+ messages in thread
From: Alvaro Herrera @ 2020-02-11 23:12 UTC (permalink / raw)
Got rid of TYPCATEGORY_MULTIRANGE. It doesn't serve any purpose that I
can see, so I just switched that to TYPCATEGORY_RANGE.
load_multirangetype_info() pointlessly open-coded
get_range_multirange_subtype(). Simplify.
Name changes:
get_multirange_subtype -> get_range_multirange_subtype
This is more in line with how we name lsyscache.c functions, given
that the cache it serves is on pg_range. (Apparently in earlier
versions of the patch there was a pg_multirange catalog, but not
anymore)
MULTIRANGETYPE -> RANGEMULTIRANGE (the syscache)
Mostly the same reasons as above. The naming also makes us move it
next to RANGETYPE; they're both in pg_range. (It was in the place
where pg_multirange would be, alphabetically.)
---
src/backend/commands/typecmds.c | 2 +-
src/backend/parser/parse_coerce.c | 10 +++++-----
src/backend/utils/cache/lsyscache.c | 10 ++++------
src/backend/utils/cache/syscache.c | 23 ++++++++++++-----------
src/backend/utils/cache/typcache.c | 15 ++-------------
src/backend/utils/fmgr/funcapi.c | 4 ++--
src/include/catalog/pg_type.dat | 12 ++++++------
src/include/catalog/pg_type.h | 1 -
src/include/utils/lsyscache.h | 2 +-
src/include/utils/syscache.h | 2 +-
src/include/utils/typcache.h | 2 +-
11 files changed, 35 insertions(+), 48 deletions(-)
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index a8dea8d58e..bd792234f8 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -1589,7 +1589,7 @@ DefineRange(CreateRangeStmt *stmt)
GetUserId(), /* owner's ID */
-1, /* internal size (always varlena) */
TYPTYPE_MULTIRANGE, /* type-type (multirange type) */
- TYPCATEGORY_MULTIRANGE, /* type-category (multirange type) */
+ TYPCATEGORY_RANGE, /* type-category (range type) */
false, /* multirange types are never preferred */
DEFAULT_TYPDELIM, /* array element delimiter */
F_MULTIRANGE_IN, /* input procedure */
diff --git a/src/backend/parser/parse_coerce.c b/src/backend/parser/parse_coerce.c
index ff1310c8f4..27bfc4bf8c 100644
--- a/src/backend/parser/parse_coerce.c
+++ b/src/backend/parser/parse_coerce.c
@@ -1599,7 +1599,7 @@ check_generic_type_consistency(const Oid *actual_arg_types,
/* Get the element type based on the multirange type, if we have one */
if (OidIsValid(multirange_typeid))
{
- multirange_typelem = get_multirange_subtype(multirange_typeid);
+ multirange_typelem = get_range_multirange_subtype(multirange_typeid);
if (!OidIsValid(multirange_typelem))
return false; /* should be a multirange, but isn't */
@@ -1938,7 +1938,7 @@ enforce_generic_type_consistency(const Oid *actual_arg_types,
}
else
{
- multirange_typelem = get_multirange_subtype(multirange_typeid);
+ multirange_typelem = get_range_multirange_subtype(multirange_typeid);
if (!OidIsValid(multirange_typelem))
ereport(ERROR,
(errcode(ERRCODE_DATATYPE_MISMATCH),
@@ -2202,7 +2202,7 @@ resolve_generic_type(Oid declared_type,
else if (context_declared_type == ANYMULTIRANGEOID)
{
Oid context_base_type = getBaseType(context_actual_type);
- Oid multirange_typelem = get_multirange_subtype(context_base_type);
+ Oid multirange_typelem = get_range_multirange_subtype(context_base_type);
Oid range_typelem = get_range_subtype(multirange_typelem);
Oid array_typeid = get_array_type(range_typelem);
@@ -2248,7 +2248,7 @@ resolve_generic_type(Oid declared_type,
{
/* Use the element type corresponding to actual type */
Oid context_base_type = getBaseType(context_actual_type);
- Oid multirange_typelem = get_multirange_subtype(context_base_type);
+ Oid multirange_typelem = get_range_multirange_subtype(context_base_type);
if (!OidIsValid(multirange_typelem))
ereport(ERROR,
@@ -2281,7 +2281,7 @@ resolve_generic_type(Oid declared_type,
else if (context_declared_type == ANYMULTIRANGEOID)
{
Oid context_base_type = getBaseType(context_actual_type);
- Oid multirange_typelem = get_multirange_subtype(context_base_type);
+ Oid multirange_typelem = get_range_multirange_subtype(context_base_type);
if (!OidIsValid(multirange_typelem))
ereport(ERROR,
diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c
index 56a145c666..7ba93c43d7 100644
--- a/src/backend/utils/cache/lsyscache.c
+++ b/src/backend/utils/cache/lsyscache.c
@@ -3133,7 +3133,7 @@ get_namespace_name_or_temp(Oid nspid)
return get_namespace_name(nspid);
}
-/* ---------- PG_RANGE CACHE ---------- */
+/* ---------- PG_RANGE CACHES ---------- */
/*
* get_range_subtype
@@ -3211,20 +3211,18 @@ get_range_multirange(Oid rangeOid)
return InvalidOid;
}
-/* ---------- PG_MULTIRANGE CACHE ---------- */
-
/*
- * get_multirange_subtype
+ * get_range_multirange_subtype
* Returns the subtype of a given multirange type
*
* Returns InvalidOid if the type is not a multirange type.
*/
Oid
-get_multirange_subtype(Oid multirangeOid)
+get_range_multirange_subtype(Oid multirangeOid)
{
HeapTuple tp;
- tp = SearchSysCache1(MULTIRANGETYPE, ObjectIdGetDatum(multirangeOid));
+ tp = SearchSysCache1(RANGEMULTIRANGE, ObjectIdGetDatum(multirangeOid));
if (HeapTupleIsValid(tp))
{
Form_pg_range rngtup = (Form_pg_range) GETSTRUCT(tp);
diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c
index fb9179c42a..7654331a59 100644
--- a/src/backend/utils/cache/syscache.c
+++ b/src/backend/utils/cache/syscache.c
@@ -508,17 +508,6 @@ static const struct cachedesc cacheinfo[] = {
},
4
},
- {RangeRelationId, /* MULTIRANGE */
- RangeMultirangeTypidIndexId,
- 1,
- {
- Anum_pg_range_rngmultitypid,
- 0,
- 0,
- 0
- },
- 4
- },
{NamespaceRelationId, /* NAMESPACENAME */
NamespaceNameIndexId,
1,
@@ -662,6 +651,18 @@ static const struct cachedesc cacheinfo[] = {
},
64
},
+ {RangeRelationId, /* RANGEMULTIRANGE */
+ RangeMultirangeTypidIndexId,
+ 1,
+ {
+ Anum_pg_range_rngmultitypid,
+ 0,
+ 0,
+ 0
+ },
+ 4
+ },
+
{RangeRelationId, /* RANGETYPE */
RangeTypidIndexId,
1,
diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c
index 6677bc8bdc..9383b24f65 100644
--- a/src/backend/utils/cache/typcache.c
+++ b/src/backend/utils/cache/typcache.c
@@ -897,7 +897,6 @@ load_rangetype_info(TypeCacheEntry *typentry)
typentry->rngelemtype = lookup_type_cache(subtypeOid, 0);
}
-
/*
* load_multirangetype_info --- helper routine to set up multirange type
* information
@@ -905,26 +904,16 @@ load_rangetype_info(TypeCacheEntry *typentry)
static void
load_multirangetype_info(TypeCacheEntry *typentry)
{
- Form_pg_range pg_range;
- HeapTuple tup;
Oid rangetypeOid;
- /* get information from pg_range */
- tup = SearchSysCache1(MULTIRANGETYPE, ObjectIdGetDatum(typentry->type_id));
- /* should not fail, since we already checked typtype ... */
- if (!HeapTupleIsValid(tup))
+ rangetypeOid = get_range_multirange_subtype(typentry->type_id);
+ if (!OidIsValid(rangetypeOid))
elog(ERROR, "cache lookup failed for multirange type %u",
typentry->type_id);
- pg_range = (Form_pg_range) GETSTRUCT(tup);
-
- rangetypeOid = pg_range->rngtypid;
-
- ReleaseSysCache(tup);
typentry->rngtype = lookup_type_cache(rangetypeOid, TYPECACHE_RANGE_INFO);
}
-
/*
* load_domaintype_info --- helper routine to set up domain constraint info
*
diff --git a/src/backend/utils/fmgr/funcapi.c b/src/backend/utils/fmgr/funcapi.c
index defc801f7a..afc617939f 100644
--- a/src/backend/utils/fmgr/funcapi.c
+++ b/src/backend/utils/fmgr/funcapi.c
@@ -683,7 +683,7 @@ resolve_polymorphic_tupdesc(TupleDesc tupdesc, oidvector *declared_args,
ANYRANGEOID);
/* check for inconsistent range and multirange results */
- rngtype = get_multirange_subtype(mltrngtype);
+ rngtype = get_range_multirange_subtype(mltrngtype);
if (OidIsValid(anyrange_type) && anyrange_type != rngtype)
return false;
@@ -1020,7 +1020,7 @@ resolve_polymorphic_argtypes(int numargs, Oid *argtypes, char *argmodes,
ANYRANGEOID);
/* check for inconsistent range and multirange results */
- rngtype = get_multirange_subtype(mltrngtype);
+ rngtype = get_range_multirange_subtype(mltrngtype);
if (OidIsValid(anyrange_type) && anyrange_type != rngtype)
return false;
diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat
index beaa70dce6..7d4bf4c5d8 100644
--- a/src/include/catalog/pg_type.dat
+++ b/src/include/catalog/pg_type.dat
@@ -489,34 +489,34 @@
# multirange types
{ oid => '8009', array_type_oid => '8010', descr => 'multirange of integers',
typname => 'int4multirange', typlen => '-1', typbyval => 'f', typtype => 'm',
- typcategory => 'M', typinput => 'multirange_in', typoutput => 'multirange_out',
+ typcategory => 'R', typinput => 'multirange_in', typoutput => 'multirange_out',
typreceive => 'multirange_recv', typsend => 'multirange_send',
typanalyze => 'multirange_typanalyze', typalign => 'i', typstorage => 'x' },
{ oid => '8011', array_type_oid => '8012', descr => 'multirange of numerics',
typname => 'nummultirange', typlen => '-1', typbyval => 'f', typtype => 'm',
- typcategory => 'M', typinput => 'multirange_in', typoutput => 'multirange_out',
+ typcategory => 'R', typinput => 'multirange_in', typoutput => 'multirange_out',
typreceive => 'multirange_recv', typsend => 'multirange_send',
typanalyze => 'multirange_typanalyze', typalign => 'i', typstorage => 'x' },
{ oid => '8013', array_type_oid => '8014',
descr => 'multirange of timestamps without time zone',
typname => 'tsmultirange', typlen => '-1', typbyval => 'f', typtype => 'm',
- typcategory => 'M', typinput => 'multirange_in', typoutput => 'multirange_out',
+ typcategory => 'R', typinput => 'multirange_in', typoutput => 'multirange_out',
typreceive => 'multirange_recv', typsend => 'multirange_send',
typanalyze => 'multirange_typanalyze', typalign => 'd', typstorage => 'x' },
{ oid => '8015', array_type_oid => '8016',
descr => 'multirange of timestamps with time zone',
typname => 'tstzmultirange', typlen => '-1', typbyval => 'f', typtype => 'm',
- typcategory => 'M', typinput => 'multirange_in', typoutput => 'multirange_out',
+ typcategory => 'R', typinput => 'multirange_in', typoutput => 'multirange_out',
typreceive => 'multirange_recv', typsend => 'multirange_send',
typanalyze => 'multirange_typanalyze', typalign => 'd', typstorage => 'x' },
{ oid => '8017', array_type_oid => '8018', descr => 'multirange of dates',
typname => 'datemultirange', typlen => '-1', typbyval => 'f', typtype => 'm',
- typcategory => 'M', typinput => 'multirange_in', typoutput => 'multirange_out',
+ typcategory => 'R', typinput => 'multirange_in', typoutput => 'multirange_out',
typreceive => 'multirange_recv', typsend => 'multirange_send',
typanalyze => 'multirange_typanalyze', typalign => 'i', typstorage => 'x' },
{ oid => '8019', array_type_oid => '8020', descr => 'multirange of bigints',
typname => 'int8multirange', typlen => '-1', typbyval => 'f', typtype => 'm',
- typcategory => 'M', typinput => 'multirange_in', typoutput => 'multirange_out',
+ typcategory => 'R', typinput => 'multirange_in', typoutput => 'multirange_out',
typreceive => 'multirange_recv', typsend => 'multirange_send',
typanalyze => 'multirange_typanalyze', typalign => 'd', typstorage => 'x' },
diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h
index 600ac58c9d..c113320b9c 100644
--- a/src/include/catalog/pg_type.h
+++ b/src/include/catalog/pg_type.h
@@ -270,7 +270,6 @@ typedef FormData_pg_type *Form_pg_type;
#define TYPCATEGORY_ENUM 'E'
#define TYPCATEGORY_GEOMETRIC 'G'
#define TYPCATEGORY_NETWORK 'I' /* think INET */
-#define TYPCATEGORY_MULTIRANGE 'M'
#define TYPCATEGORY_NUMERIC 'N'
#define TYPCATEGORY_PSEUDOTYPE 'P'
#define TYPCATEGORY_RANGE 'R'
diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h
index 7d1674a3b6..fffd4c5a45 100644
--- a/src/include/utils/lsyscache.h
+++ b/src/include/utils/lsyscache.h
@@ -182,7 +182,7 @@ extern char *get_namespace_name_or_temp(Oid nspid);
extern Oid get_range_subtype(Oid rangeOid);
extern Oid get_range_collation(Oid rangeOid);
extern Oid get_range_multirange(Oid rangeOid);
-extern Oid get_multirange_subtype(Oid multirangeOid);
+extern Oid get_range_multirange_subtype(Oid multirangeOid);
extern Oid get_index_column_opclass(Oid index_oid, int attno);
#define type_is_array(typid) (get_element_type(typid) != InvalidOid)
diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h
index bc51bbeb54..e8f393520a 100644
--- a/src/include/utils/syscache.h
+++ b/src/include/utils/syscache.h
@@ -66,7 +66,6 @@ enum SysCacheIdentifier
INDEXRELID,
LANGNAME,
LANGOID,
- MULTIRANGETYPE,
NAMESPACENAME,
NAMESPACEOID,
OPERNAMENSP,
@@ -80,6 +79,7 @@ enum SysCacheIdentifier
PUBLICATIONOID,
PUBLICATIONREL,
PUBLICATIONRELMAP,
+ RANGEMULTIRANGE,
RANGETYPE,
RELNAMENSP,
RELOID,
diff --git a/src/include/utils/typcache.h b/src/include/utils/typcache.h
index fe240ea004..9957be4640 100644
--- a/src/include/utils/typcache.h
+++ b/src/include/utils/typcache.h
@@ -101,7 +101,7 @@ typedef struct TypeCacheEntry
/*
* Fields computed when TYPCACHE_MULTIRANGE_INFO is required.
*/
- struct TypeCacheEntry *rngtype;
+ struct TypeCacheEntry *rngtype; /* underlying range type */
/*
* Domain's base type and typmod if it's a domain type. Zeroes if not
--
2.20.1
--wRRV7LY7NUeQGEoC--
^ permalink raw reply [nested|flat] 6+ messages in thread
* [PATCH v5 3/5] fixup: Streaming Read API
@ 2024-02-26 10:48 Thomas Munro <[email protected]>
0 siblings, 0 replies; 6+ messages in thread
From: Thomas Munro @ 2024-02-26 10:48 UTC (permalink / raw)
---
contrib/pg_prewarm/pg_prewarm.c | 40 +-
src/backend/storage/Makefile | 2 +-
src/backend/storage/aio/Makefile | 14 +
src/backend/storage/aio/meson.build | 5 +
src/backend/storage/aio/streaming_read.c | 611 +++++++++++++++++++++
src/backend/storage/buffer/bufmgr.c | 641 ++++++++++++++++-------
src/backend/storage/buffer/localbuf.c | 14 +-
src/backend/storage/meson.build | 1 +
src/include/storage/bufmgr.h | 45 ++
src/include/storage/streaming_read.h | 52 ++
src/tools/pgindent/typedefs.list | 3 +
11 files changed, 1217 insertions(+), 211 deletions(-)
create mode 100644 src/backend/storage/aio/Makefile
create mode 100644 src/backend/storage/aio/meson.build
create mode 100644 src/backend/storage/aio/streaming_read.c
create mode 100644 src/include/storage/streaming_read.h
diff --git a/contrib/pg_prewarm/pg_prewarm.c b/contrib/pg_prewarm/pg_prewarm.c
index 8541e4d6e46..1cc84bcb0c2 100644
--- a/contrib/pg_prewarm/pg_prewarm.c
+++ b/contrib/pg_prewarm/pg_prewarm.c
@@ -20,6 +20,7 @@
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "storage/smgr.h"
+#include "storage/streaming_read.h"
#include "utils/acl.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
@@ -38,6 +39,25 @@ typedef enum
static PGIOAlignedBlock blockbuffer;
+struct pg_prewarm_streaming_read_private
+{
+ BlockNumber blocknum;
+ int64 last_block;
+};
+
+static BlockNumber
+pg_prewarm_streaming_read_next(PgStreamingRead *pgsr,
+ void *pgsr_private,
+ void *per_buffer_data)
+{
+ struct pg_prewarm_streaming_read_private *p = pgsr_private;
+
+ if (p->blocknum <= p->last_block)
+ return p->blocknum++;
+
+ return InvalidBlockNumber;
+}
+
/*
* pg_prewarm(regclass, mode text, fork text,
* first_block int8, last_block int8)
@@ -183,18 +203,36 @@ pg_prewarm(PG_FUNCTION_ARGS)
}
else if (ptype == PREWARM_BUFFER)
{
+ struct pg_prewarm_streaming_read_private p;
+ PgStreamingRead *pgsr;
+
/*
* In buffer mode, we actually pull the data into shared_buffers.
*/
+
+ /* Set up the private state for our streaming buffer read callback. */
+ p.blocknum = first_block;
+ p.last_block = last_block;
+
+ pgsr = pg_streaming_read_buffer_alloc(PGSR_FLAG_FULL,
+ &p,
+ 0,
+ NULL,
+ BMR_REL(rel),
+ forkNumber,
+ pg_prewarm_streaming_read_next);
+
for (block = first_block; block <= last_block; ++block)
{
Buffer buf;
CHECK_FOR_INTERRUPTS();
- buf = ReadBufferExtended(rel, forkNumber, block, RBM_NORMAL, NULL);
+ buf = pg_streaming_read_buffer_get_next(pgsr, NULL);
ReleaseBuffer(buf);
++blocks_done;
}
+ Assert(pg_streaming_read_buffer_get_next(pgsr, NULL) == InvalidBuffer);
+ pg_streaming_read_free(pgsr);
}
/* Close relation, release lock. */
diff --git a/src/backend/storage/Makefile b/src/backend/storage/Makefile
index 8376cdfca20..eec03f6f2b4 100644
--- a/src/backend/storage/Makefile
+++ b/src/backend/storage/Makefile
@@ -8,6 +8,6 @@ subdir = src/backend/storage
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
-SUBDIRS = buffer file freespace ipc large_object lmgr page smgr sync
+SUBDIRS = aio buffer file freespace ipc large_object lmgr page smgr sync
include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/storage/aio/Makefile b/src/backend/storage/aio/Makefile
new file mode 100644
index 00000000000..bcab44c802f
--- /dev/null
+++ b/src/backend/storage/aio/Makefile
@@ -0,0 +1,14 @@
+#
+# Makefile for storage/aio
+#
+# src/backend/storage/aio/Makefile
+#
+
+subdir = src/backend/storage/aio
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = \
+ streaming_read.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/storage/aio/meson.build b/src/backend/storage/aio/meson.build
new file mode 100644
index 00000000000..39aef2a84a2
--- /dev/null
+++ b/src/backend/storage/aio/meson.build
@@ -0,0 +1,5 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+backend_sources += files(
+ 'streaming_read.c',
+)
diff --git a/src/backend/storage/aio/streaming_read.c b/src/backend/storage/aio/streaming_read.c
new file mode 100644
index 00000000000..e437abebe4c
--- /dev/null
+++ b/src/backend/storage/aio/streaming_read.c
@@ -0,0 +1,611 @@
+#include "postgres.h"
+
+#include "storage/streaming_read.h"
+#include "utils/rel.h"
+
+/*
+ * Element type for PgStreamingRead's circular array of block ranges.
+ */
+typedef struct PgStreamingReadRange
+{
+ bool need_wait;
+ bool advice_issued;
+ BlockNumber blocknum;
+ int nblocks;
+ int per_buffer_data_index;
+ Buffer buffers[MAX_BUFFERS_PER_TRANSFER];
+ ReadBuffersOperation operation;
+} PgStreamingReadRange;
+
+/*
+ * Streaming read object.
+ */
+struct PgStreamingRead
+{
+ int max_ios;
+ int ios_in_progress;
+ int max_pinned_buffers;
+ int pinned_buffers;
+ int pinned_buffers_trigger;
+ int next_tail_buffer;
+ int distance;
+ bool finished;
+ bool advice_enabled;
+ void *pgsr_private;
+ PgStreamingReadBufferCB callback;
+
+ BufferAccessStrategy strategy;
+ BufferManagerRelation bmr;
+ ForkNumber forknum;
+
+ /* Sometimes we need to buffer one block for flow control. */
+ BlockNumber unget_blocknum;
+ void *unget_per_buffer_data;
+
+ /* Next expected block, for detecting sequential access. */
+ BlockNumber seq_blocknum;
+
+ /* Space for optional per-buffer private data. */
+ size_t per_buffer_data_size;
+ void *per_buffer_data;
+
+ /* Circular buffer of ranges. */
+ int size;
+ int head;
+ int tail;
+ PgStreamingReadRange ranges[FLEXIBLE_ARRAY_MEMBER];
+};
+
+static PgStreamingRead *
+pg_streaming_read_buffer_alloc_internal(int flags,
+ void *pgsr_private,
+ size_t per_buffer_data_size,
+ BufferAccessStrategy strategy)
+{
+ PgStreamingRead *pgsr;
+ int size;
+ int max_ios;
+ uint32 max_pinned_buffers;
+
+
+ /*
+ * Decide how many assumed I/Os we will allow to run concurrently. That
+ * is, advice to the kernel to tell it that we will soon read. This
+ * number also affects how far we look ahead for opportunities to start
+ * more I/Os.
+ */
+ if (flags & PGSR_FLAG_MAINTENANCE)
+ max_ios = maintenance_io_concurrency;
+ else
+ max_ios = effective_io_concurrency;
+
+ /*
+ * The desired level of I/O concurrency controls how far ahead we are
+ * willing to look ahead. We also clamp it to at least
+ * MAX_BUFFER_PER_TRANFER so that we can have a chance to build up a full
+ * sized read, even when max_ios is zero.
+ */
+ max_pinned_buffers = Max(max_ios * 4, MAX_BUFFERS_PER_TRANSFER);
+
+ /*
+ * The *_io_concurrency GUCs might be set to 0, but we want to allow at
+ * least one, to keep our gating logic simple.
+ */
+ max_ios = Max(max_ios, 1);
+
+ /*
+ * Don't allow this backend to pin too many buffers. For now we'll apply
+ * the limit for the shared buffer pool and the local buffer pool, without
+ * worrying which it is.
+ */
+ LimitAdditionalPins(&max_pinned_buffers);
+ LimitAdditionalLocalPins(&max_pinned_buffers);
+ Assert(max_pinned_buffers > 0);
+
+ /*
+ * pgsr->ranges is a circular buffer. When it is empty, head == tail.
+ * When it is full, there is an empty element between head and tail. Head
+ * can also be empty (nblocks == 0), therefore we need two extra elements
+ * for non-occupied ranges, on top of max_pinned_buffers to allow for the
+ * maxmimum possible number of occupied ranges of the smallest possible
+ * size of one.
+ */
+ size = max_pinned_buffers + 2;
+
+ pgsr = (PgStreamingRead *)
+ palloc0(offsetof(PgStreamingRead, ranges) +
+ sizeof(pgsr->ranges[0]) * size);
+
+ pgsr->max_ios = max_ios;
+ pgsr->per_buffer_data_size = per_buffer_data_size;
+ pgsr->max_pinned_buffers = max_pinned_buffers;
+ pgsr->pgsr_private = pgsr_private;
+ pgsr->strategy = strategy;
+ pgsr->size = size;
+
+ pgsr->unget_blocknum = InvalidBlockNumber;
+
+#ifdef USE_PREFETCH
+
+ /*
+ * This system supports prefetching advice. As long as direct I/O isn't
+ * enabled, and the caller hasn't promised sequential access, we can use
+ * it.
+ */
+ if ((io_direct_flags & IO_DIRECT_DATA) == 0 &&
+ (flags & PGSR_FLAG_SEQUENTIAL) == 0)
+ pgsr->advice_enabled = true;
+#endif
+
+ /*
+ * Skip the initial ramp-up phase if the caller says we're going to be
+ * reading the whole relation. This way we start out doing full-sized
+ * reads.
+ */
+ if (flags & PGSR_FLAG_FULL)
+ pgsr->distance = Min(MAX_BUFFERS_PER_TRANSFER, pgsr->max_pinned_buffers);
+ else
+ pgsr->distance = 1;
+
+ /*
+ * We want to avoid creating ranges that are smaller than they could be
+ * just because we hit max_pinned_buffers. We only look ahead when the
+ * number of pinned buffers falls below this trigger number, or put
+ * another way, we stop looking ahead when we wouldn't be able to build a
+ * "full sized" range.
+ */
+ pgsr->pinned_buffers_trigger =
+ Max(1, (int) max_pinned_buffers - MAX_BUFFERS_PER_TRANSFER);
+
+ /* Space for the callback to store extra data along with each block. */
+ if (per_buffer_data_size)
+ pgsr->per_buffer_data = palloc(per_buffer_data_size * max_pinned_buffers);
+
+ return pgsr;
+}
+
+/*
+ * Create a new streaming read object that can be used to perform the
+ * equivalent of a series of ReadBuffer() calls for one fork of one relation.
+ * Internally, it generates larger vectored reads where possible by looking
+ * ahead.
+ */
+PgStreamingRead *
+pg_streaming_read_buffer_alloc(int flags,
+ void *pgsr_private,
+ size_t per_buffer_data_size,
+ BufferAccessStrategy strategy,
+ BufferManagerRelation bmr,
+ ForkNumber forknum,
+ PgStreamingReadBufferCB next_block_cb)
+{
+ PgStreamingRead *result;
+
+ result = pg_streaming_read_buffer_alloc_internal(flags,
+ pgsr_private,
+ per_buffer_data_size,
+ strategy);
+ result->callback = next_block_cb;
+ result->bmr = bmr;
+ result->forknum = forknum;
+
+ return result;
+}
+
+/*
+ * Find the per-buffer data index for the Nth block of a range.
+ */
+static int
+get_per_buffer_data_index(PgStreamingRead *pgsr, PgStreamingReadRange *range, int n)
+{
+ int result;
+
+ /*
+ * Find slot in the circular buffer of per-buffer data, without using the
+ * expensive % operator.
+ */
+ result = range->per_buffer_data_index + n;
+ if (result >= pgsr->max_pinned_buffers)
+ result -= pgsr->max_pinned_buffers;
+ Assert(result == (range->per_buffer_data_index + n) % pgsr->max_pinned_buffers);
+
+ return result;
+}
+
+/*
+ * Return a pointer to the per-buffer data by index.
+ */
+static void *
+get_per_buffer_data_by_index(PgStreamingRead *pgsr, int per_buffer_data_index)
+{
+ return (char *) pgsr->per_buffer_data +
+ pgsr->per_buffer_data_size * per_buffer_data_index;
+}
+
+/*
+ * Return a pointer to the per-buffer data for the Nth block of a range.
+ */
+static void *
+get_per_buffer_data(PgStreamingRead *pgsr, PgStreamingReadRange *range, int n)
+{
+ return get_per_buffer_data_by_index(pgsr,
+ get_per_buffer_data_index(pgsr,
+ range,
+ n));
+}
+
+/*
+ * Start reading the head range, and create a new head range. The new head
+ * range is returned. It may not be empty, if StartReadBuffers() couldn't
+ * start the entire range; in that case the returned range contains the
+ * remaining portion of the range.
+ */
+static PgStreamingReadRange *
+pg_streaming_read_start_head_range(PgStreamingRead *pgsr)
+{
+ PgStreamingReadRange *head_range;
+ PgStreamingReadRange *new_head_range;
+ int nblocks_pinned;
+ int flags;
+
+ /* Caller should make sure we never exceed max_ios. */
+ Assert(pgsr->ios_in_progress < pgsr->max_ios);
+
+ /* Should only call if the head range has some blocks to read. */
+ head_range = &pgsr->ranges[pgsr->head];
+ Assert(head_range->nblocks > 0);
+
+ /*
+ * If advice hasn't been suppressed, and this system supports it, this
+ * isn't a strictly sequential pattern, then we'll issue advice.
+ */
+ if (pgsr->advice_enabled && head_range->blocknum != pgsr->seq_blocknum)
+ flags = READ_BUFFERS_ISSUE_ADVICE;
+ else
+ flags = 0;
+
+
+ /* Start reading as many blocks as we can from the head range. */
+ nblocks_pinned = head_range->nblocks;
+ head_range->need_wait =
+ StartReadBuffers(pgsr->bmr,
+ head_range->buffers,
+ pgsr->forknum,
+ head_range->blocknum,
+ &nblocks_pinned,
+ pgsr->strategy,
+ flags,
+ &head_range->operation);
+
+ if (head_range->need_wait)
+ {
+ /*
+ * I/O necessary. Look-ahead distance increases rapidly until it hits
+ * the pin limit.
+ */
+ if (pgsr->distance < pgsr->max_pinned_buffers)
+ {
+ int distance;
+
+ distance = pgsr->distance * 2;
+ distance = Min(distance, pgsr->max_pinned_buffers);
+ pgsr->distance = distance;
+ }
+
+ /* Count an I/O in progress until we've "waited". */
+ if (flags & READ_BUFFERS_ISSUE_ADVICE)
+ {
+ head_range->advice_issued = true;
+ pgsr->ios_in_progress++;
+ Assert(pgsr->ios_in_progress <= pgsr->max_ios);
+ }
+ }
+ else
+ {
+ /*
+ * No I/O necessary. Look-ahead distance decays slowly, but stays high
+ * enough to form a full sized I/O.
+ */
+ if (pgsr->distance > MAX_BUFFERS_PER_TRANSFER)
+ pgsr->distance--;
+ }
+
+ /*
+ * StartReadBuffers() might have pinned fewer blocks than we asked it to,
+ * but always at least one.
+ */
+ Assert(nblocks_pinned <= head_range->nblocks);
+ Assert(nblocks_pinned >= 1);
+ pgsr->pinned_buffers += nblocks_pinned;
+
+ /*
+ * Remember where the next block would be after that, so we can detect
+ * sequential access next time.
+ */
+ pgsr->seq_blocknum = head_range->blocknum + nblocks_pinned;
+
+ /*
+ * Create a new head range. There must be space, because we have enough
+ * elements for every range to hold just one block, up to the pin limit.
+ */
+ Assert(pgsr->size > pgsr->max_pinned_buffers);
+ Assert((pgsr->head + 1) % pgsr->size != pgsr->tail);
+ if (++pgsr->head == pgsr->size)
+ pgsr->head = 0;
+ new_head_range = &pgsr->ranges[pgsr->head];
+ new_head_range->nblocks = 0;
+ new_head_range->advice_issued = false;
+
+ /*
+ * If we didn't manage to start the whole read above, we split the range,
+ * moving the remainder into the new head range.
+ */
+ if (nblocks_pinned < head_range->nblocks)
+ {
+ int nblocks_remaining = head_range->nblocks - nblocks_pinned;
+
+ head_range->nblocks = nblocks_pinned;
+
+ new_head_range->blocknum = head_range->blocknum + nblocks_pinned;
+ new_head_range->nblocks = nblocks_remaining;
+ }
+
+ /* The new range has per-buffer data starting after the previous range. */
+ new_head_range->per_buffer_data_index =
+ get_per_buffer_data_index(pgsr, head_range, nblocks_pinned);
+
+ return new_head_range;
+}
+
+/*
+ * Ask the callback which block it would like us to read next, with a small
+ * buffer in front to allow pg_streaming_unget_block() to work.
+ */
+static BlockNumber
+pg_streaming_get_block(PgStreamingRead *pgsr, void *per_buffer_data)
+{
+ BlockNumber result;
+
+ if (unlikely(pgsr->unget_blocknum != InvalidBlockNumber))
+ {
+ /*
+ * If we had to unget a block, now it is time to return that one
+ * again.
+ */
+ result = pgsr->unget_blocknum;
+ pgsr->unget_blocknum = InvalidBlockNumber;
+
+ /*
+ * The same per_buffer_data element must have been used, and still
+ * contains whatever data the callback wrote into it. So we just
+ * sanity-check that we were called with the value that
+ * pg_streaming_unget_block() pushed back.
+ */
+ Assert(per_buffer_data == pgsr->unget_per_buffer_data);
+ }
+ else
+ {
+ /* Use the installed callback directly. */
+ result = pgsr->callback(pgsr, pgsr->pgsr_private, per_buffer_data);
+ }
+
+ return result;
+}
+
+/*
+ * In order to deal with short reads in StartReadBuffers(), we sometimes need
+ * to defer handling of a block until later. This *must* be called with the
+ * last value returned by pg_streaming_get_block().
+ */
+static void
+pg_streaming_unget_block(PgStreamingRead *pgsr, BlockNumber blocknum, void *per_buffer_data)
+{
+ Assert(pgsr->unget_blocknum == InvalidBlockNumber);
+ pgsr->unget_blocknum = blocknum;
+ pgsr->unget_per_buffer_data = per_buffer_data;
+}
+
+static void
+pg_streaming_read_look_ahead(PgStreamingRead *pgsr)
+{
+ PgStreamingReadRange *range;
+
+ /*
+ * If we're finished or can't start more I/O, then don't look ahead.
+ */
+ if (pgsr->finished || pgsr->ios_in_progress == pgsr->max_ios)
+ return;
+
+ /*
+ * We'll also wait until the number of pinned buffers falls below our
+ * trigger level, so that we have the chance to create a full range.
+ */
+ if (pgsr->pinned_buffers >= pgsr->pinned_buffers_trigger)
+ return;
+
+ do
+ {
+ BlockNumber blocknum;
+ void *per_buffer_data;
+
+ /* Do we have a full-sized range? */
+ range = &pgsr->ranges[pgsr->head];
+ if (range->nblocks == lengthof(range->buffers))
+ {
+ /* Start as much of it as we can. */
+ range = pg_streaming_read_start_head_range(pgsr);
+
+ /* If we're now at the I/O limit, stop here. */
+ if (pgsr->ios_in_progress == pgsr->max_ios)
+ return;
+
+ /*
+ * If we couldn't form a full range, then stop here to avoid
+ * creating small I/O.
+ */
+ if (pgsr->pinned_buffers >= pgsr->pinned_buffers_trigger)
+ return;
+
+ /*
+ * That might have only been partially started, but always
+ * processes at least one so that'll do for now.
+ */
+ Assert(range->nblocks < lengthof(range->buffers));
+ }
+
+ /* Find per-buffer data slot for the next block. */
+ per_buffer_data = get_per_buffer_data(pgsr, range, range->nblocks);
+
+ /* Find out which block the callback wants to read next. */
+ blocknum = pg_streaming_get_block(pgsr, per_buffer_data);
+ if (blocknum == InvalidBlockNumber)
+ {
+ /* End of stream. */
+ pgsr->finished = true;
+ break;
+ }
+
+ /*
+ * Is there a head range that we cannot extend, because the requested
+ * block is not consecutive?
+ */
+ if (range->nblocks > 0 &&
+ range->blocknum + range->nblocks != blocknum)
+ {
+ /* Yes. Start it, so we can begin building a new one. */
+ range = pg_streaming_read_start_head_range(pgsr);
+
+ /*
+ * It's possible that it was only partially started, and we have a
+ * new range with the remainder. Keep starting I/Os until we get
+ * it all out of the way, or we hit the I/O limit.
+ */
+ while (range->nblocks > 0 && pgsr->ios_in_progress < pgsr->max_ios)
+ range = pg_streaming_read_start_head_range(pgsr);
+
+ /*
+ * We have to 'unget' the block returned by the callback if we
+ * don't have enough I/O capacity left to start something.
+ */
+ if (pgsr->ios_in_progress == pgsr->max_ios)
+ {
+ pg_streaming_unget_block(pgsr, blocknum, per_buffer_data);
+ return;
+ }
+ }
+
+ /* If we have a new, empty range, initialize the start block. */
+ if (range->nblocks == 0)
+ {
+ range->blocknum = blocknum;
+ }
+
+ /* This block extends the range by one. */
+ Assert(range->blocknum + range->nblocks == blocknum);
+ range->nblocks++;
+
+ } while (pgsr->pinned_buffers + range->nblocks < pgsr->distance);
+
+ /* Start as much as we can. */
+ while (range->nblocks > 0)
+ {
+ range = pg_streaming_read_start_head_range(pgsr);
+ if (pgsr->ios_in_progress == pgsr->max_ios)
+ break;
+ }
+}
+
+Buffer
+pg_streaming_read_buffer_get_next(PgStreamingRead *pgsr, void **per_buffer_data)
+{
+ pg_streaming_read_look_ahead(pgsr);
+
+ /* See if we have one buffer to return. */
+ while (pgsr->tail != pgsr->head)
+ {
+ PgStreamingReadRange *tail_range;
+
+ tail_range = &pgsr->ranges[pgsr->tail];
+
+ /*
+ * Do we need to perform an I/O before returning the buffers from this
+ * range?
+ */
+ if (tail_range->need_wait)
+ {
+ WaitReadBuffers(&tail_range->operation);
+ tail_range->need_wait = false;
+
+ /*
+ * We don't really know if the kernel generated a physical I/O
+ * when we issued advice, let alone when it finished, but it has
+ * certainly finished now because we've performed the read.
+ */
+ if (tail_range->advice_issued)
+ {
+ Assert(pgsr->ios_in_progress > 0);
+ pgsr->ios_in_progress--;
+ }
+ }
+
+ /* Are there more buffers available in this range? */
+ if (pgsr->next_tail_buffer < tail_range->nblocks)
+ {
+ int buffer_index;
+ Buffer buffer;
+
+ buffer_index = pgsr->next_tail_buffer++;
+ buffer = tail_range->buffers[buffer_index];
+
+ Assert(BufferIsValid(buffer));
+
+ /* We are giving away ownership of this pinned buffer. */
+ Assert(pgsr->pinned_buffers > 0);
+ pgsr->pinned_buffers--;
+
+ if (per_buffer_data)
+ *per_buffer_data = get_per_buffer_data(pgsr, tail_range, buffer_index);
+
+ return buffer;
+ }
+
+ /* Advance tail to next range, if there is one. */
+ if (++pgsr->tail == pgsr->size)
+ pgsr->tail = 0;
+ pgsr->next_tail_buffer = 0;
+
+ /*
+ * If tail crashed into head, and head is not empty, then it is time
+ * to start that range.
+ */
+ if (pgsr->tail == pgsr->head &&
+ pgsr->ranges[pgsr->head].nblocks > 0)
+ pg_streaming_read_start_head_range(pgsr);
+ }
+
+ Assert(pgsr->pinned_buffers == 0);
+
+ return InvalidBuffer;
+}
+
+void
+pg_streaming_read_free(PgStreamingRead *pgsr)
+{
+ Buffer buffer;
+
+ /* Stop looking ahead. */
+ pgsr->finished = true;
+
+ /* Unpin anything that wasn't consumed. */
+ while ((buffer = pg_streaming_read_buffer_get_next(pgsr, NULL)) != InvalidBuffer)
+ ReleaseBuffer(buffer);
+
+ Assert(pgsr->pinned_buffers == 0);
+ Assert(pgsr->ios_in_progress == 0);
+
+ /* Release memory. */
+ if (pgsr->per_buffer_data)
+ pfree(pgsr->per_buffer_data);
+
+ pfree(pgsr);
+}
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index f0f8d4259c5..729d1f91721 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -19,6 +19,11 @@
* and pin it so that no one can destroy it while this process
* is using it.
*
+ * StartReadBuffers() -- as above, but for multiple contiguous blocks in
+ * two steps.
+ *
+ * WaitReadBuffers() -- second step of StartReadBuffers().
+ *
* ReleaseBuffer() -- unpin a buffer
*
* MarkBufferDirty() -- mark a pinned buffer's contents as "dirty".
@@ -471,10 +476,9 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref)
)
-static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence,
+static Buffer ReadBuffer_common(BufferManagerRelation bmr,
ForkNumber forkNum, BlockNumber blockNum,
- ReadBufferMode mode, BufferAccessStrategy strategy,
- bool *hit);
+ ReadBufferMode mode, BufferAccessStrategy strategy);
static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr,
ForkNumber fork,
BufferAccessStrategy strategy,
@@ -500,7 +504,7 @@ static uint32 WaitBufHdrUnlocked(BufferDesc *buf);
static int SyncOneBuffer(int buf_id, bool skip_recently_used,
WritebackContext *wb_context);
static void WaitIO(BufferDesc *buf);
-static bool StartBufferIO(BufferDesc *buf, bool forInput);
+static bool StartBufferIO(BufferDesc *buf, bool forInput, bool nowait);
static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty,
uint32 set_flag_bits, bool forget_owner);
static void AbortBufferIO(Buffer buffer);
@@ -781,7 +785,6 @@ Buffer
ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum,
ReadBufferMode mode, BufferAccessStrategy strategy)
{
- bool hit;
Buffer buf;
/*
@@ -794,15 +797,9 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("cannot access temporary tables of other sessions")));
- /*
- * Read the buffer, and update pgstat counters to reflect a cache hit or
- * miss.
- */
- pgstat_count_buffer_read(reln);
- buf = ReadBuffer_common(RelationGetSmgr(reln), reln->rd_rel->relpersistence,
- forkNum, blockNum, mode, strategy, &hit);
- if (hit)
- pgstat_count_buffer_hit(reln);
+ buf = ReadBuffer_common(BMR_REL(reln),
+ forkNum, blockNum, mode, strategy);
+
return buf;
}
@@ -822,13 +819,12 @@ ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum,
BlockNumber blockNum, ReadBufferMode mode,
BufferAccessStrategy strategy, bool permanent)
{
- bool hit;
-
SMgrRelation smgr = smgropen(rlocator, INVALID_PROC_NUMBER);
- return ReadBuffer_common(smgr, permanent ? RELPERSISTENCE_PERMANENT :
- RELPERSISTENCE_UNLOGGED, forkNum, blockNum,
- mode, strategy, &hit);
+ return ReadBuffer_common(BMR_SMGR(smgr, permanent ? RELPERSISTENCE_PERMANENT :
+ RELPERSISTENCE_UNLOGGED),
+ forkNum, blockNum,
+ mode, strategy);
}
/*
@@ -994,35 +990,68 @@ ExtendBufferedRelTo(BufferManagerRelation bmr,
*/
if (buffer == InvalidBuffer)
{
- bool hit;
-
Assert(extended_by == 0);
- buffer = ReadBuffer_common(bmr.smgr, bmr.relpersistence,
- fork, extend_to - 1, mode, strategy,
- &hit);
+ buffer = ReadBuffer_common(bmr, fork, extend_to - 1, mode, strategy);
}
return buffer;
}
+/*
+ * Zero a buffer and lock it, as part of the implementation of
+ * RBM_ZERO_AND_LOCK or RBM_ZERO_AND_CLEANUP_LOCK. The buffer must be already
+ * pinned. It does not have to be valid, but it is valid and locked on
+ * return.
+ */
+static void
+ZeroBuffer(Buffer buffer, ReadBufferMode mode)
+{
+ BufferDesc *bufHdr;
+ uint32 buf_state;
+
+ Assert(mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK);
+
+ if (BufferIsLocal(buffer))
+ bufHdr = GetLocalBufferDescriptor(-buffer - 1);
+ else
+ {
+ bufHdr = GetBufferDescriptor(buffer - 1);
+ if (mode == RBM_ZERO_AND_LOCK)
+ LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
+ else
+ LockBufferForCleanup(buffer);
+ }
+
+ memset(BufferGetPage(buffer), 0, BLCKSZ);
+
+ if (BufferIsLocal(buffer))
+ {
+ buf_state = pg_atomic_read_u32(&bufHdr->state);
+ buf_state |= BM_VALID;
+ pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
+ }
+ else
+ {
+ buf_state = LockBufHdr(bufHdr);
+ buf_state |= BM_VALID;
+ UnlockBufHdr(bufHdr, buf_state);
+ }
+}
+
/*
* ReadBuffer_common -- common logic for all ReadBuffer variants
*
* *hit is set to true if the request was satisfied from shared buffer cache.
*/
static Buffer
-ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
+ReadBuffer_common(BufferManagerRelation bmr, ForkNumber forkNum,
BlockNumber blockNum, ReadBufferMode mode,
- BufferAccessStrategy strategy, bool *hit)
+ BufferAccessStrategy strategy)
{
- BufferDesc *bufHdr;
- Block bufBlock;
- bool found;
- IOContext io_context;
- IOObject io_object;
- bool isLocalBuf = SmgrIsTemp(smgr);
-
- *hit = false;
+ ReadBuffersOperation operation;
+ Buffer buffer;
+ int nblocks;
+ int flags;
/*
* Backward compatibility path, most code should use ExtendBufferedRel()
@@ -1041,181 +1070,404 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK)
flags |= EB_LOCK_FIRST;
- return ExtendBufferedRel(BMR_SMGR(smgr, relpersistence),
- forkNum, strategy, flags);
+ return ExtendBufferedRel(bmr, forkNum, strategy, flags);
}
- TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum,
- smgr->smgr_rlocator.locator.spcOid,
- smgr->smgr_rlocator.locator.dbOid,
- smgr->smgr_rlocator.locator.relNumber,
- smgr->smgr_rlocator.backend);
+ nblocks = 1;
+ if (mode == RBM_ZERO_ON_ERROR)
+ flags = READ_BUFFERS_ZERO_ON_ERROR;
+ else
+ flags = 0;
+ if (StartReadBuffers(bmr,
+ &buffer,
+ forkNum,
+ blockNum,
+ &nblocks,
+ strategy,
+ flags,
+ &operation))
+ WaitReadBuffers(&operation);
+ Assert(nblocks == 1); /* single block can't be short */
+
+ if (mode == RBM_ZERO_AND_CLEANUP_LOCK || mode == RBM_ZERO_AND_LOCK)
+ ZeroBuffer(buffer, mode);
+
+ return buffer;
+}
+
+static Buffer
+PrepareReadBuffer(BufferManagerRelation bmr,
+ ForkNumber forkNum,
+ BlockNumber blockNum,
+ BufferAccessStrategy strategy,
+ bool *foundPtr)
+{
+ BufferDesc *bufHdr;
+ bool isLocalBuf;
+ IOContext io_context;
+ IOObject io_object;
+
+ Assert(blockNum != P_NEW);
+ Assert(bmr.smgr);
+
+ isLocalBuf = SmgrIsTemp(bmr.smgr);
if (isLocalBuf)
{
- /*
- * We do not use a BufferAccessStrategy for I/O of temporary tables.
- * However, in some cases, the "strategy" may not be NULL, so we can't
- * rely on IOContextForStrategy() to set the right IOContext for us.
- * This may happen in cases like CREATE TEMPORARY TABLE AS...
- */
io_context = IOCONTEXT_NORMAL;
io_object = IOOBJECT_TEMP_RELATION;
- bufHdr = LocalBufferAlloc(smgr, forkNum, blockNum, &found);
- if (found)
- pgBufferUsage.local_blks_hit++;
- else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG ||
- mode == RBM_ZERO_ON_ERROR)
- pgBufferUsage.local_blks_read++;
}
else
{
- /*
- * lookup the buffer. IO_IN_PROGRESS is set if the requested block is
- * not currently in memory.
- */
io_context = IOContextForStrategy(strategy);
io_object = IOOBJECT_RELATION;
- bufHdr = BufferAlloc(smgr, relpersistence, forkNum, blockNum,
- strategy, &found, io_context);
- if (found)
- pgBufferUsage.shared_blks_hit++;
- else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG ||
- mode == RBM_ZERO_ON_ERROR)
- pgBufferUsage.shared_blks_read++;
}
- /* At this point we do NOT hold any locks. */
+ TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum,
+ bmr.smgr->smgr_rlocator.locator.spcOid,
+ bmr.smgr->smgr_rlocator.locator.dbOid,
+ bmr.smgr->smgr_rlocator.locator.relNumber,
+ bmr.smgr->smgr_rlocator.backend);
- /* if it was already in the buffer pool, we're done */
- if (found)
+ ResourceOwnerEnlarge(CurrentResourceOwner);
+ if (isLocalBuf)
+ {
+ bufHdr = LocalBufferAlloc(bmr.smgr, forkNum, blockNum, foundPtr);
+ if (*foundPtr)
+ pgBufferUsage.local_blks_hit++;
+ }
+ else
+ {
+ bufHdr = BufferAlloc(bmr.smgr, bmr.relpersistence, forkNum, blockNum,
+ strategy, foundPtr, io_context);
+ if (*foundPtr)
+ pgBufferUsage.shared_blks_hit++;
+ }
+ if (bmr.rel)
+ {
+ /*
+ * While pgBufferUsage's "read" counter isn't bumped unless we reach
+ * WaitReadBuffers() (so, not for hits, and not for buffers that are
+ * zeroed instead), the per-relation stats always count them.
+ */
+ pgstat_count_buffer_read(bmr.rel);
+ if (*foundPtr)
+ pgstat_count_buffer_hit(bmr.rel);
+ }
+ if (*foundPtr)
{
- /* Just need to update stats before we exit */
- *hit = true;
VacuumPageHit++;
pgstat_count_io_op(io_object, io_context, IOOP_HIT);
-
if (VacuumCostActive)
VacuumCostBalance += VacuumCostPageHit;
TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum,
- smgr->smgr_rlocator.locator.spcOid,
- smgr->smgr_rlocator.locator.dbOid,
- smgr->smgr_rlocator.locator.relNumber,
- smgr->smgr_rlocator.backend,
- found);
+ bmr.smgr->smgr_rlocator.locator.spcOid,
+ bmr.smgr->smgr_rlocator.locator.dbOid,
+ bmr.smgr->smgr_rlocator.locator.relNumber,
+ bmr.smgr->smgr_rlocator.backend,
+ true);
+ }
- /*
- * In RBM_ZERO_AND_LOCK mode the caller expects the page to be locked
- * on return.
- */
- if (!isLocalBuf)
- {
- if (mode == RBM_ZERO_AND_LOCK)
- LWLockAcquire(BufferDescriptorGetContentLock(bufHdr),
- LW_EXCLUSIVE);
- else if (mode == RBM_ZERO_AND_CLEANUP_LOCK)
- LockBufferForCleanup(BufferDescriptorGetBuffer(bufHdr));
- }
+ return BufferDescriptorGetBuffer(bufHdr);
+}
- return BufferDescriptorGetBuffer(bufHdr);
+/*
+ * Begin reading a range of blocks beginning at blockNum and extending for
+ * *nblocks. On return, up to *nblocks pinned buffers holding those blocks
+ * are written into the buffers array, and *nblocks is updated to contain the
+ * actual number, which may be fewer than requested.
+ *
+ * If false is returned, no I/O is necessary and WaitReadBuffers() is not
+ * necessary. If true is returned, one I/O has been started, and
+ * WaitReadBuffers() must be called with the same operation object before the
+ * buffers are accessed. Along with the operation object, the caller-supplied
+ * array of buffers must remain valid until WaitReadBuffers() is called.
+ *
+ * Currently the I/O is only started with optional operating system advice,
+ * and the real I/O happens in WaitReadBuffers(). In future work, true I/O
+ * could be initiated here.
+ */
+bool
+StartReadBuffers(BufferManagerRelation bmr,
+ Buffer *buffers,
+ ForkNumber forkNum,
+ BlockNumber blockNum,
+ int *nblocks,
+ BufferAccessStrategy strategy,
+ int flags,
+ ReadBuffersOperation *operation)
+{
+ int actual_nblocks = *nblocks;
+
+ if (bmr.rel)
+ {
+ bmr.smgr = RelationGetSmgr(bmr.rel);
+ bmr.relpersistence = bmr.rel->rd_rel->relpersistence;
}
- /*
- * if we have gotten to this point, we have allocated a buffer for the
- * page but its contents are not yet valid. IO_IN_PROGRESS is set for it,
- * if it's a shared buffer.
- */
- Assert(!(pg_atomic_read_u32(&bufHdr->state) & BM_VALID)); /* spinlock not needed */
+ operation->bmr = bmr;
+ operation->forknum = forkNum;
+ operation->blocknum = blockNum;
+ operation->buffers = buffers;
+ operation->nblocks = actual_nblocks;
+ operation->strategy = strategy;
+ operation->flags = flags;
- bufBlock = isLocalBuf ? LocalBufHdrGetBlock(bufHdr) : BufHdrGetBlock(bufHdr);
+ operation->io_buffers_len = 0;
- /*
- * Read in the page, unless the caller intends to overwrite it and just
- * wants us to allocate a buffer.
- */
- if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK)
- MemSet((char *) bufBlock, 0, BLCKSZ);
- else
+ for (int i = 0; i < actual_nblocks; ++i)
{
- instr_time io_start = pgstat_prepare_io_time(track_io_timing);
+ bool found;
- smgrread(smgr, forkNum, blockNum, bufBlock);
+ buffers[i] = PrepareReadBuffer(bmr,
+ forkNum,
+ blockNum + i,
+ strategy,
+ &found);
- pgstat_count_io_op_time(io_object, io_context,
- IOOP_READ, io_start, 1);
+ if (found)
+ {
+ /*
+ * Terminate the read as soon as we get a hit. It could be a
+ * single buffer hit, or it could be a hit that follows a readable
+ * range. We don't want to create more than one readable range,
+ * so we stop here.
+ */
+ actual_nblocks = operation->nblocks = *nblocks = i + 1;
+ }
+ else
+ {
+ /* Extend the readable range to cover this block. */
+ operation->io_buffers_len++;
+ }
+ }
- /* check for garbage data */
- if (!PageIsVerifiedExtended((Page) bufBlock, blockNum,
- PIV_LOG_WARNING | PIV_REPORT_STAT))
+ if (operation->io_buffers_len > 0)
+ {
+ if (flags & READ_BUFFERS_ISSUE_ADVICE)
{
- if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages)
- {
- ereport(WARNING,
- (errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("invalid page in block %u of relation %s; zeroing out page",
- blockNum,
- relpath(smgr->smgr_rlocator, forkNum))));
- MemSet((char *) bufBlock, 0, BLCKSZ);
- }
- else
- ereport(ERROR,
- (errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("invalid page in block %u of relation %s",
- blockNum,
- relpath(smgr->smgr_rlocator, forkNum))));
+ /*
+ * In theory we should only do this if PrepareReadBuffers() had to
+ * allocate new buffers above. That way, if two calls to
+ * StartReadBuffers() were made for the same blocks before
+ * WaitReadBuffers(), only the first would issue the advice.
+ * That'd be a better simulation of true asynchronous I/O, which
+ * would only start the I/O once, but isn't done here for
+ * simplicity. Note also that the following call might actually
+ * issue two advice calls if we cross a segment boundary; in a
+ * true asynchronous version we might choose to process only one
+ * real I/O at a time in that case.
+ */
+ smgrprefetch(bmr.smgr, forkNum, blockNum, operation->io_buffers_len);
}
+
+ /* Indicate that WaitReadBuffers() should be called. */
+ return true;
}
+ else
+ {
+ return false;
+ }
+}
- /*
- * In RBM_ZERO_AND_LOCK / RBM_ZERO_AND_CLEANUP_LOCK mode, grab the buffer
- * content lock before marking the page as valid, to make sure that no
- * other backend sees the zeroed page before the caller has had a chance
- * to initialize it.
- *
- * Since no-one else can be looking at the page contents yet, there is no
- * difference between an exclusive lock and a cleanup-strength lock. (Note
- * that we cannot use LockBuffer() or LockBufferForCleanup() here, because
- * they assert that the buffer is already valid.)
- */
- if ((mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) &&
- !isLocalBuf)
+static inline bool
+WaitReadBuffersCanStartIO(Buffer buffer, bool nowait)
+{
+ if (BufferIsLocal(buffer))
{
- LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE);
+ BufferDesc *bufHdr = GetLocalBufferDescriptor(-buffer - 1);
+
+ return (pg_atomic_read_u32(&bufHdr->state) & BM_VALID) == 0;
}
+ else
+ return StartBufferIO(GetBufferDescriptor(buffer - 1), true, nowait);
+}
+
+void
+WaitReadBuffers(ReadBuffersOperation *operation)
+{
+ BufferManagerRelation bmr;
+ Buffer *buffers;
+ int nblocks;
+ BlockNumber blocknum;
+ ForkNumber forknum;
+ bool isLocalBuf;
+ IOContext io_context;
+ IOObject io_object;
+
+ /*
+ * Currently operations are only allowed to include a read of some range,
+ * with an optional extra buffer that is already pinned at the end. So
+ * nblocks can be at most one more than io_buffers_len.
+ */
+ Assert((operation->nblocks == operation->io_buffers_len) ||
+ (operation->nblocks == operation->io_buffers_len + 1));
+ /* Find the range of the physical read we need to perform. */
+ nblocks = operation->io_buffers_len;
+ if (nblocks == 0)
+ return; /* nothing to do */
+
+ buffers = &operation->buffers[0];
+ blocknum = operation->blocknum;
+ forknum = operation->forknum;
+ bmr = operation->bmr;
+
+ isLocalBuf = SmgrIsTemp(bmr.smgr);
if (isLocalBuf)
{
- /* Only need to adjust flags */
- uint32 buf_state = pg_atomic_read_u32(&bufHdr->state);
-
- buf_state |= BM_VALID;
- pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
+ io_context = IOCONTEXT_NORMAL;
+ io_object = IOOBJECT_TEMP_RELATION;
}
else
{
- /* Set BM_VALID, terminate IO, and wake up any waiters */
- TerminateBufferIO(bufHdr, false, BM_VALID, true);
+ io_context = IOContextForStrategy(operation->strategy);
+ io_object = IOOBJECT_RELATION;
}
- VacuumPageMiss++;
- if (VacuumCostActive)
- VacuumCostBalance += VacuumCostPageMiss;
+ /*
+ * We count all these blocks as read by this backend. This is traditional
+ * behavior, but might turn out to be not true if we find that someone
+ * else has beaten us and completed the read of some of these blocks. In
+ * that case the system globally double-counts, but we traditionally don't
+ * count this as a "hit", and we don't have a separate counter for "miss,
+ * but another backend completed the read".
+ */
+ if (isLocalBuf)
+ pgBufferUsage.local_blks_read += nblocks;
+ else
+ pgBufferUsage.shared_blks_read += nblocks;
- TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum,
- smgr->smgr_rlocator.locator.spcOid,
- smgr->smgr_rlocator.locator.dbOid,
- smgr->smgr_rlocator.locator.relNumber,
- smgr->smgr_rlocator.backend,
- found);
+ for (int i = 0; i < nblocks; ++i)
+ {
+ int io_buffers_len;
+ Buffer io_buffers[MAX_BUFFERS_PER_TRANSFER];
+ void *io_pages[MAX_BUFFERS_PER_TRANSFER];
+ instr_time io_start;
+ BlockNumber io_first_block;
- return BufferDescriptorGetBuffer(bufHdr);
+ /*
+ * Skip this block if someone else has already completed it. If an
+ * I/O is already in progress in another backend, this will wait for
+ * the outcome: either done, or something went wrong and we will
+ * retry.
+ */
+ if (!WaitReadBuffersCanStartIO(buffers[i], false))
+ {
+ /*
+ * Report this as a 'hit' for this backend, even though it must
+ * have started out as a miss in PrepareReadBuffer().
+ */
+ TRACE_POSTGRESQL_BUFFER_READ_DONE(forknum, blocknum + i,
+ bmr.smgr->smgr_rlocator.locator.spcOid,
+ bmr.smgr->smgr_rlocator.locator.dbOid,
+ bmr.smgr->smgr_rlocator.locator.relNumber,
+ bmr.smgr->smgr_rlocator.backend,
+ true);
+ continue;
+ }
+
+ /* We found a buffer that we need to read in. */
+ io_buffers[0] = buffers[i];
+ io_pages[0] = BufferGetBlock(buffers[i]);
+ io_first_block = blocknum + i;
+ io_buffers_len = 1;
+
+ /*
+ * How many neighboring-on-disk blocks can we can scatter-read into
+ * other buffers at the same time? In this case we don't wait if we
+ * see an I/O already in progress. We already hold BM_IO_IN_PROGRESS
+ * for the head block, so we should get on with that I/O as soon as
+ * possible. We'll come back to this block again, above.
+ */
+ while ((i + 1) < nblocks &&
+ WaitReadBuffersCanStartIO(buffers[i + 1], true))
+ {
+ /* Must be consecutive block numbers. */
+ Assert(BufferGetBlockNumber(buffers[i + 1]) ==
+ BufferGetBlockNumber(buffers[i]) + 1);
+
+ io_buffers[io_buffers_len] = buffers[++i];
+ io_pages[io_buffers_len++] = BufferGetBlock(buffers[i]);
+ }
+
+ io_start = pgstat_prepare_io_time(track_io_timing);
+ smgrreadv(bmr.smgr, forknum, io_first_block, io_pages, io_buffers_len);
+ pgstat_count_io_op_time(io_object, io_context, IOOP_READ, io_start,
+ io_buffers_len);
+
+ /* Verify each block we read, and terminate the I/O. */
+ for (int j = 0; j < io_buffers_len; ++j)
+ {
+ BufferDesc *bufHdr;
+ Block bufBlock;
+
+ if (isLocalBuf)
+ {
+ bufHdr = GetLocalBufferDescriptor(-io_buffers[j] - 1);
+ bufBlock = LocalBufHdrGetBlock(bufHdr);
+ }
+ else
+ {
+ bufHdr = GetBufferDescriptor(io_buffers[j] - 1);
+ bufBlock = BufHdrGetBlock(bufHdr);
+ }
+
+ /* check for garbage data */
+ if (!PageIsVerifiedExtended((Page) bufBlock, io_first_block + j,
+ PIV_LOG_WARNING | PIV_REPORT_STAT))
+ {
+ if ((operation->flags & READ_BUFFERS_ZERO_ON_ERROR) || zero_damaged_pages)
+ {
+ ereport(WARNING,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("invalid page in block %u of relation %s; zeroing out page",
+ io_first_block + j,
+ relpath(bmr.smgr->smgr_rlocator, forknum))));
+ memset(bufBlock, 0, BLCKSZ);
+ }
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("invalid page in block %u of relation %s",
+ io_first_block + j,
+ relpath(bmr.smgr->smgr_rlocator, forknum))));
+ }
+
+ /* Terminate I/O and set BM_VALID. */
+ if (isLocalBuf)
+ {
+ uint32 buf_state = pg_atomic_read_u32(&bufHdr->state);
+
+ buf_state |= BM_VALID;
+ pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
+ }
+ else
+ {
+ /* Set BM_VALID, terminate IO, and wake up any waiters */
+ TerminateBufferIO(bufHdr, false, BM_VALID, true);
+ }
+
+ /* Report I/Os as completing individually. */
+ TRACE_POSTGRESQL_BUFFER_READ_DONE(forknum, io_first_block + j,
+ bmr.smgr->smgr_rlocator.locator.spcOid,
+ bmr.smgr->smgr_rlocator.locator.dbOid,
+ bmr.smgr->smgr_rlocator.locator.relNumber,
+ bmr.smgr->smgr_rlocator.backend,
+ false);
+ }
+
+ VacuumPageMiss += io_buffers_len;
+ if (VacuumCostActive)
+ VacuumCostBalance += VacuumCostPageMiss * io_buffers_len;
+ }
}
/*
- * BufferAlloc -- subroutine for ReadBuffer. Handles lookup of a shared
- * buffer. If no buffer exists already, selects a replacement
- * victim and evicts the old page, but does NOT read in new page.
+ * BufferAlloc -- subroutine for StartReadBuffers. Handles lookup of a shared
+ * buffer. If no buffer exists already, selects a replacement victim and
+ * evicts the old page, but does NOT read in new page.
*
* "strategy" can be a buffer replacement strategy object, or NULL for
* the default strategy. The selected buffer's usage_count is advanced when
@@ -1223,11 +1475,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
*
* The returned buffer is pinned and is already marked as holding the
* desired page. If it already did have the desired page, *foundPtr is
- * set true. Otherwise, *foundPtr is set false and the buffer is marked
- * as IO_IN_PROGRESS; ReadBuffer will now need to do I/O to fill it.
- *
- * *foundPtr is actually redundant with the buffer's BM_VALID flag, but
- * we keep it for simplicity in ReadBuffer.
+ * set true. Otherwise, *foundPtr is set false.
*
* io_context is passed as an output parameter to avoid calling
* IOContextForStrategy() when there is a shared buffers hit and no IO
@@ -1286,19 +1534,10 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
{
/*
* We can only get here if (a) someone else is still reading in
- * the page, or (b) a previous read attempt failed. We have to
- * wait for any active read attempt to finish, and then set up our
- * own read attempt if the page is still not BM_VALID.
- * StartBufferIO does it all.
+ * the page, (b) a previous read attempt failed, or (c) someone
+ * called StartReadBuffers() but not yet WaitReadBuffers().
*/
- if (StartBufferIO(buf, true))
- {
- /*
- * If we get here, previous attempts to read the buffer must
- * have failed ... but we shall bravely try again.
- */
- *foundPtr = false;
- }
+ *foundPtr = false;
}
return buf;
@@ -1363,19 +1602,10 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
{
/*
* We can only get here if (a) someone else is still reading in
- * the page, or (b) a previous read attempt failed. We have to
- * wait for any active read attempt to finish, and then set up our
- * own read attempt if the page is still not BM_VALID.
- * StartBufferIO does it all.
+ * the page, (b) a previous read attempt failed, or (c) someone
+ * called StartReadBuffers() but not yet WaitReadBuffers().
*/
- if (StartBufferIO(existing_buf_hdr, true))
- {
- /*
- * If we get here, previous attempts to read the buffer must
- * have failed ... but we shall bravely try again.
- */
- *foundPtr = false;
- }
+ *foundPtr = false;
}
return existing_buf_hdr;
@@ -1407,15 +1637,9 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
LWLockRelease(newPartitionLock);
/*
- * Buffer contents are currently invalid. Try to obtain the right to
- * start I/O. If StartBufferIO returns false, then someone else managed
- * to read it before we did, so there's nothing left for BufferAlloc() to
- * do.
+ * Buffer contents are currently invalid.
*/
- if (StartBufferIO(victim_buf_hdr, true))
- *foundPtr = false;
- else
- *foundPtr = true;
+ *foundPtr = false;
return victim_buf_hdr;
}
@@ -1769,7 +1993,7 @@ again:
* pessimistic, but outside of toy-sized shared_buffers it should allow
* sufficient pins.
*/
-static void
+void
LimitAdditionalPins(uint32 *additional_pins)
{
uint32 max_backends;
@@ -2034,7 +2258,7 @@ ExtendBufferedRelShared(BufferManagerRelation bmr,
buf_state &= ~BM_VALID;
UnlockBufHdr(existing_hdr, buf_state);
- } while (!StartBufferIO(existing_hdr, true));
+ } while (!StartBufferIO(existing_hdr, true, false));
}
else
{
@@ -2057,7 +2281,7 @@ ExtendBufferedRelShared(BufferManagerRelation bmr,
LWLockRelease(partition_lock);
/* XXX: could combine the locked operations in it with the above */
- StartBufferIO(victim_buf_hdr, true);
+ StartBufferIO(victim_buf_hdr, true, false);
}
}
@@ -2372,7 +2596,12 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy)
else
{
/*
- * If we previously pinned the buffer, it must surely be valid.
+ * If we previously pinned the buffer, it is likely to be valid, but
+ * it may not be if StartReadBuffers() was called and
+ * WaitReadBuffers() hasn't been called yet. We'll check by loading
+ * the flags without locking. This is racy, but it's OK to return
+ * false spuriously: when WaitReadBuffers() calls StartBufferIO(),
+ * it'll see that it's now valid.
*
* Note: We deliberately avoid a Valgrind client request here.
* Individual access methods can optionally superimpose buffer page
@@ -2381,7 +2610,7 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy)
* that the buffer page is legitimately non-accessible here. We
* cannot meddle with that.
*/
- result = true;
+ result = (pg_atomic_read_u32(&buf->state) & BM_VALID) != 0;
}
ref->refcount++;
@@ -3449,7 +3678,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object,
* someone else flushed the buffer before we could, so we need not do
* anything.
*/
- if (!StartBufferIO(buf, false))
+ if (!StartBufferIO(buf, false, false))
return;
/* Setup error traceback support for ereport() */
@@ -5184,9 +5413,15 @@ WaitIO(BufferDesc *buf)
*
* Returns true if we successfully marked the buffer as I/O busy,
* false if someone else already did the work.
+ *
+ * If nowait is true, then we don't wait for an I/O to be finished by another
+ * backend. In that case, false indicates either that the I/O was already
+ * finished, or is still in progress. This is useful for callers that want to
+ * find out if they can perform the I/O as part of a larger operation, without
+ * waiting for the answer or distinguishing the reasons why not.
*/
static bool
-StartBufferIO(BufferDesc *buf, bool forInput)
+StartBufferIO(BufferDesc *buf, bool forInput, bool nowait)
{
uint32 buf_state;
@@ -5199,6 +5434,8 @@ StartBufferIO(BufferDesc *buf, bool forInput)
if (!(buf_state & BM_IO_IN_PROGRESS))
break;
UnlockBufHdr(buf, buf_state);
+ if (nowait)
+ return false;
WaitIO(buf);
}
diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c
index fcfac335a57..985a2c7049c 100644
--- a/src/backend/storage/buffer/localbuf.c
+++ b/src/backend/storage/buffer/localbuf.c
@@ -108,10 +108,9 @@ PrefetchLocalBuffer(SMgrRelation smgr, ForkNumber forkNum,
* LocalBufferAlloc -
* Find or create a local buffer for the given page of the given relation.
*
- * API is similar to bufmgr.c's BufferAlloc, except that we do not need
- * to do any locking since this is all local. Also, IO_IN_PROGRESS
- * does not get set. Lastly, we support only default access strategy
- * (hence, usage_count is always advanced).
+ * API is similar to bufmgr.c's BufferAlloc, except that we do not need to do
+ * any locking since this is all local. We support only default access
+ * strategy (hence, usage_count is always advanced).
*/
BufferDesc *
LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum,
@@ -287,7 +286,7 @@ GetLocalVictimBuffer(void)
}
/* see LimitAdditionalPins() */
-static void
+void
LimitAdditionalLocalPins(uint32 *additional_pins)
{
uint32 max_pins;
@@ -297,9 +296,10 @@ LimitAdditionalLocalPins(uint32 *additional_pins)
/*
* In contrast to LimitAdditionalPins() other backends don't play a role
- * here. We can allow up to NLocBuffer pins in total.
+ * here. We can allow up to NLocBuffer pins in total, but it might not be
+ * initialized yet so read num_temp_buffers.
*/
- max_pins = (NLocBuffer - NLocalPinnedBuffers);
+ max_pins = (num_temp_buffers - NLocalPinnedBuffers);
if (*additional_pins >= max_pins)
*additional_pins = max_pins;
diff --git a/src/backend/storage/meson.build b/src/backend/storage/meson.build
index 40345bdca27..739d13293fb 100644
--- a/src/backend/storage/meson.build
+++ b/src/backend/storage/meson.build
@@ -1,5 +1,6 @@
# Copyright (c) 2022-2024, PostgreSQL Global Development Group
+subdir('aio')
subdir('buffer')
subdir('file')
subdir('freespace')
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index d51d46d3353..b57f71f97e3 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -14,6 +14,7 @@
#ifndef BUFMGR_H
#define BUFMGR_H
+#include "port/pg_iovec.h"
#include "storage/block.h"
#include "storage/buf.h"
#include "storage/bufpage.h"
@@ -158,6 +159,11 @@ extern PGDLLIMPORT int32 *LocalRefCount;
#define BUFFER_LOCK_SHARE 1
#define BUFFER_LOCK_EXCLUSIVE 2
+/*
+ * Maximum number of buffers for multi-buffer I/O functions. This is set to
+ * allow 128kB transfers, unless BLCKSZ and IOV_MAX imply a a smaller maximum.
+ */
+#define MAX_BUFFERS_PER_TRANSFER Min(PG_IOV_MAX, (128 * 1024) / BLCKSZ)
/*
* prototypes for functions in bufmgr.c
@@ -177,6 +183,42 @@ extern Buffer ReadBufferWithoutRelcache(RelFileLocator rlocator,
ForkNumber forkNum, BlockNumber blockNum,
ReadBufferMode mode, BufferAccessStrategy strategy,
bool permanent);
+
+#define READ_BUFFERS_ZERO_ON_ERROR 0x01
+#define READ_BUFFERS_ISSUE_ADVICE 0x02
+
+/*
+ * Private state used by StartReadBuffers() and WaitReadBuffers(). Declared
+ * in public header only to allow inclusion in other structs, but contents
+ * should not be accessed.
+ */
+struct ReadBuffersOperation
+{
+ /* Parameters passed in to StartReadBuffers(). */
+ BufferManagerRelation bmr;
+ Buffer *buffers;
+ ForkNumber forknum;
+ BlockNumber blocknum;
+ int nblocks;
+ BufferAccessStrategy strategy;
+ int flags;
+
+ /* Range of buffers, if we need to perform a read. */
+ int io_buffers_len;
+};
+
+typedef struct ReadBuffersOperation ReadBuffersOperation;
+
+extern bool StartReadBuffers(BufferManagerRelation bmr,
+ Buffer *buffers,
+ ForkNumber forknum,
+ BlockNumber blocknum,
+ int *nblocks,
+ BufferAccessStrategy strategy,
+ int flags,
+ ReadBuffersOperation *operation);
+extern void WaitReadBuffers(ReadBuffersOperation *operation);
+
extern void ReleaseBuffer(Buffer buffer);
extern void UnlockReleaseBuffer(Buffer buffer);
extern bool BufferIsExclusiveLocked(Buffer buffer);
@@ -250,6 +292,9 @@ extern bool HoldingBufferPinThatDelaysRecovery(void);
extern bool BgBufferSync(struct WritebackContext *wb_context);
+extern void LimitAdditionalPins(uint32 *additional_pins);
+extern void LimitAdditionalLocalPins(uint32 *additional_pins);
+
/* in buf_init.c */
extern void InitBufferPool(void);
extern Size BufferShmemSize(void);
diff --git a/src/include/storage/streaming_read.h b/src/include/storage/streaming_read.h
new file mode 100644
index 00000000000..c4d3892bb26
--- /dev/null
+++ b/src/include/storage/streaming_read.h
@@ -0,0 +1,52 @@
+#ifndef STREAMING_READ_H
+#define STREAMING_READ_H
+
+#include "storage/bufmgr.h"
+#include "storage/fd.h"
+#include "storage/smgr.h"
+
+/* Default tuning, reasonable for many users. */
+#define PGSR_FLAG_DEFAULT 0x00
+
+/*
+ * I/O streams that are performing maintenance work on behalf of potentially
+ * many users.
+ */
+#define PGSR_FLAG_MAINTENANCE 0x01
+
+/*
+ * We usually avoid issuing prefetch advice automatically when sequential
+ * access is detected, but this flag explicitly disables it, for cases that
+ * might not be correctly detected. Explicit advice is known to perform worse
+ * than letting the kernel (at least Linux) detect sequential access.
+ */
+#define PGSR_FLAG_SEQUENTIAL 0x02
+
+/*
+ * We usually ramp up from smaller reads to larger ones, to support users who
+ * don't know if it's worth reading lots of buffers yet. This flag disables
+ * that, declaring ahead of time that we'll be reading all available buffers.
+ */
+#define PGSR_FLAG_FULL 0x04
+
+struct PgStreamingRead;
+typedef struct PgStreamingRead PgStreamingRead;
+
+/* Callback that returns the next block number to read. */
+typedef BlockNumber (*PgStreamingReadBufferCB) (PgStreamingRead *pgsr,
+ void *pgsr_private,
+ void *per_buffer_private);
+
+extern PgStreamingRead *pg_streaming_read_buffer_alloc(int flags,
+ void *pgsr_private,
+ size_t per_buffer_private_size,
+ BufferAccessStrategy strategy,
+ BufferManagerRelation bmr,
+ ForkNumber forknum,
+ PgStreamingReadBufferCB next_block_cb);
+
+extern void pg_streaming_read_prefetch(PgStreamingRead *pgsr);
+extern Buffer pg_streaming_read_buffer_get_next(PgStreamingRead *pgsr, void **per_buffer_private);
+extern void pg_streaming_read_free(PgStreamingRead *pgsr);
+
+#endif
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index d3a7f75b080..299c77ea69f 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2097,6 +2097,8 @@ PgStat_TableCounts
PgStat_TableStatus
PgStat_TableXactStatus
PgStat_WalStats
+PgStreamingRead
+PgStreamingReadRange
PgXmlErrorContext
PgXmlStrictness
Pg_finfo_record
@@ -2267,6 +2269,7 @@ ReInitializeDSMForeignScan_function
ReScanForeignScan_function
ReadBufPtrType
ReadBufferMode
+ReadBuffersOperation
ReadBytePtrType
ReadExtraTocPtrType
ReadFunc
--
2.40.1
--kqbls4vrrkltflt4
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v5-0004-Add-pg_streaming_read_reset.patch"
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: miscellaneous pg_upgrade cleanup
@ 2024-09-23 13:04 Daniel Gustafsson <[email protected]>
0 siblings, 1 reply; 6+ messages in thread
From: Daniel Gustafsson @ 2024-09-23 13:04 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: pgsql-hackers
> On 17 Sep 2024, at 21:22, Nathan Bossart <[email protected]> wrote:
>
> Here are a few miscellaneous cleanup patches for pg_upgrade. I don't think
> there's anything controversial here.
No objections to any of these changes, LGTM.
--
Daniel Gustafsson
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: miscellaneous pg_upgrade cleanup
@ 2024-09-23 14:06 Nathan Bossart <[email protected]>
parent: Daniel Gustafsson <[email protected]>
0 siblings, 0 replies; 6+ messages in thread
From: Nathan Bossart @ 2024-09-23 14:06 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: pgsql-hackers
On Mon, Sep 23, 2024 at 03:04:22PM +0200, Daniel Gustafsson wrote:
> No objections to any of these changes, LGTM.
Thanks for reviewing. I'll commit these once the v17 release freeze is
over (since 0001 needs to be back-patched there).
--
nathan
^ permalink raw reply [nested|flat] 6+ messages in thread
end of thread, other threads:[~2024-09-23 14:06 UTC | newest]
Thread overview: 6+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2018-06-18 09:25 [PATCH 1/8] Make syntax changes for custom compression methods Ildus Kurbangaliev <[email protected]>
2018-06-18 09:25 [PATCH 1/8] Make syntax changes for custom compression methods Ildus Kurbangaliev <[email protected]>
2020-02-11 23:12 [PATCH v11 4/4] Various tweaks, mostly naming Alvaro Herrera <[email protected]>
2024-02-26 10:48 [PATCH v5 3/5] fixup: Streaming Read API Thomas Munro <[email protected]>
2024-09-23 13:04 Re: miscellaneous pg_upgrade cleanup Daniel Gustafsson <[email protected]>
2024-09-23 14:06 ` Re: miscellaneous pg_upgrade cleanup Nathan Bossart <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox