public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v49 2/7] Add conditional lock feature to dshash
6+ messages / 4 participants
[nested] [flat]
* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 6+ messages in thread
From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)
Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
src/include/lib/dshash.h | 3 ++
2 files changed, 55 insertions(+), 46 deletions(-)
diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
* the caller must take care to ensure that the entry is not left corrupted.
* The lock mode is either shared or exclusive depending on 'exclusive'.
*
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
* The caller must not lock a lock already.
*
* Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
void *
dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
{
- dshash_hash hash;
- size_t partition;
- dshash_table_item *item;
-
- hash = hash_key(hash_table, key);
- partition = PARTITION_FOR_HASH(hash);
-
- Assert(hash_table->control->magic == DSHASH_MAGIC);
- Assert(!hash_table->find_locked);
-
- LWLockAcquire(PARTITION_LOCK(hash_table, partition),
- exclusive ? LW_EXCLUSIVE : LW_SHARED);
- ensure_valid_bucket_pointers(hash_table);
-
- /* Search the active bucket. */
- item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
- if (!item)
- {
- /* Not found. */
- LWLockRelease(PARTITION_LOCK(hash_table, partition));
- return NULL;
- }
- else
- {
- /* The caller will free the lock by calling dshash_release_lock. */
- hash_table->find_locked = true;
- hash_table->find_exclusively_locked = exclusive;
- return ENTRY_FROM_ITEM(item);
- }
+ return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
}
/*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
const void *key,
bool *found)
{
- dshash_hash hash;
- size_t partition_index;
- dshash_partition *partition;
+ return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked. If "nowait" is true, the function immediately returns if
+ * required lock was not acquired. "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+ bool exclusive, bool nowait, bool insert, bool *found)
+{
+ dshash_hash hash = hash_key(hash_table, key);
+ size_t partidx = PARTITION_FOR_HASH(hash);
+ dshash_partition *partition = &hash_table->control->partitions[partidx];
+ LWLockMode lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
dshash_table_item *item;
- hash = hash_key(hash_table, key);
- partition_index = PARTITION_FOR_HASH(hash);
- partition = &hash_table->control->partitions[partition_index];
-
- Assert(hash_table->control->magic == DSHASH_MAGIC);
- Assert(!hash_table->find_locked);
+ /* must be exclusive when insert allowed */
+ Assert(!insert || (exclusive && found != NULL));
restart:
- LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
- LW_EXCLUSIVE);
+ if (!nowait)
+ LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+ else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+ lockmode))
+ return NULL;
+
ensure_valid_bucket_pointers(hash_table);
/* Search the active bucket. */
item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
if (item)
- *found = true;
+ {
+ if (found)
+ *found = true;
+ }
else
{
- *found = false;
+ if (found)
+ *found = false;
+
+ if (!insert)
+ {
+ /* The caller didn't told to add a new entry. */
+ LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+ return NULL;
+ }
/* Check if we are getting too full. */
if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
* Give up our existing lock first, because resizing needs to
* reacquire all the locks in the right order to avoid deadlocks.
*/
- LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+ LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
resize(hash_table, hash_table->size_log2 + 1);
goto restart;
@@ -493,12 +498,13 @@ restart:
++partition->count;
}
- /* The caller must release the lock with dshash_release_lock. */
+ /* The caller will free the lock by calling dshash_release_lock. */
hash_table->find_locked = true;
- hash_table->find_exclusively_locked = true;
+ hash_table->find_exclusively_locked = exclusive;
return ENTRY_FROM_ITEM(item);
}
+
/*
* Remove an entry by key. Returns true if the key was found and the
* corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
const void *key, bool exclusive);
extern void *dshash_find_or_insert(dshash_table *hash_table,
const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+ bool exclusive, bool nowait, bool insert,
+ bool *found);
extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
extern void dshash_release_lock(dshash_table *hash_table, void *entry);
--
2.27.0
----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"
^ permalink raw reply [nested|flat] 6+ messages in thread
* Disabling Heap-Only Tuples
@ 2023-07-05 10:44 Thom Brown <[email protected]>
2023-07-05 10:57 ` Re: Disabling Heap-Only Tuples Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 6+ messages in thread
From: Thom Brown @ 2023-07-05 10:44 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>
Hi,
Heap-Only Tuple (HOT) updates are a significant performance
enhancement, as they prevent unnecessary page writes. However, HOT
comes with a caveat: it means that if we have lots of available space
earlier on in the relation, it can only be used for new tuples or in
cases where there's insufficient space on a page for an UPDATE to use
HOT.
This mechanism limits our options for condensing tables, forcing us to
resort to methods like running VACUUM FULL/CLUSTER or using external
tools like pg_repack. These either require exclusive locks (which will
be a deal-breaker on large tables on a production system), or there's
risks involved. Of course we can always flood pages with new versions
of a row until it's forced onto an early page, but that shouldn't be
necessary.
Considering these trade-offs, I'd like to propose an option to allow
superusers to disable HOT on tables. The intent is to trade some
performance benefits for the ability to reduce the size of a table
without the typical locking associated with it.
This feature could be used to shrink tables in one of two ways:
temporarily disabling HOT until DML operations have compacted the data
into a smaller area, or performing a mass update on later rows to
relocate them to an earlier location, probably in stages. Of course,
this would need to be used in conjunction with a VACUUM operation.
Admittedly this isn't ideal, and it would be better if we had an
operation that could do this (e.g. VACUUM COMPACT <table_name>), or an
option that causes some operations to avoid HOT when it detects an
amount of free space over a threshold, but in lieu of those, I thought
this would at least allow users to help themselves when running into
disk space issues.
Thoughts?
Thom
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: Disabling Heap-Only Tuples
2023-07-05 10:44 Disabling Heap-Only Tuples Thom Brown <[email protected]>
@ 2023-07-05 10:57 ` Matthias van de Meent <[email protected]>
2023-07-05 11:02 ` Re: Disabling Heap-Only Tuples Thom Brown <[email protected]>
0 siblings, 1 reply; 6+ messages in thread
From: Matthias van de Meent @ 2023-07-05 10:57 UTC (permalink / raw)
To: Thom Brown <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Wed, 5 Jul 2023 at 12:45, Thom Brown <[email protected]> wrote:
> Heap-Only Tuple (HOT) updates are a significant performance
> enhancement, as they prevent unnecessary page writes. However, HOT
> comes with a caveat: it means that if we have lots of available space
> earlier on in the relation, it can only be used for new tuples or in
> cases where there's insufficient space on a page for an UPDATE to use
> HOT.
>
> This mechanism limits our options for condensing tables, forcing us to
> resort to methods like running VACUUM FULL/CLUSTER or using external
> tools like pg_repack. These either require exclusive locks (which will
> be a deal-breaker on large tables on a production system), or there's
> risks involved. Of course we can always flood pages with new versions
> of a row until it's forced onto an early page, but that shouldn't be
> necessary.
>
> Considering these trade-offs, I'd like to propose an option to allow
> superusers to disable HOT on tables. The intent is to trade some
> performance benefits for the ability to reduce the size of a table
> without the typical locking associated with it.
Interesting use case, but I think that disabling HOT would be missing
the forest for the trees. I think that a feature that disables
block-local updates for pages > some offset would be a better solution
to your issue: Normal updates also prefer the new tuple to be stored
in the same pages as the old tuple if at all possible, so disabling
HOT wouldn't solve the issue of tuples residing in the tail of your
table - at least not while there is still empty space in those pages.
Kind regards,
Matthias van de Meent
Neon (https://neon.tech/)
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: Disabling Heap-Only Tuples
2023-07-05 10:44 Disabling Heap-Only Tuples Thom Brown <[email protected]>
2023-07-05 10:57 ` Re: Disabling Heap-Only Tuples Matthias van de Meent <[email protected]>
@ 2023-07-05 11:02 ` Thom Brown <[email protected]>
2023-07-05 12:12 ` Re: Disabling Heap-Only Tuples Matthias van de Meent <[email protected]>
2023-07-05 19:47 ` Re: Disabling Heap-Only Tuples Laurenz Albe <[email protected]>
0 siblings, 2 replies; 6+ messages in thread
From: Thom Brown @ 2023-07-05 11:02 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Wed, 5 Jul 2023 at 11:57, Matthias van de Meent
<[email protected]> wrote:
>
> On Wed, 5 Jul 2023 at 12:45, Thom Brown <[email protected]> wrote:
> > Heap-Only Tuple (HOT) updates are a significant performance
> > enhancement, as they prevent unnecessary page writes. However, HOT
> > comes with a caveat: it means that if we have lots of available space
> > earlier on in the relation, it can only be used for new tuples or in
> > cases where there's insufficient space on a page for an UPDATE to use
> > HOT.
> >
> > This mechanism limits our options for condensing tables, forcing us to
> > resort to methods like running VACUUM FULL/CLUSTER or using external
> > tools like pg_repack. These either require exclusive locks (which will
> > be a deal-breaker on large tables on a production system), or there's
> > risks involved. Of course we can always flood pages with new versions
> > of a row until it's forced onto an early page, but that shouldn't be
> > necessary.
> >
> > Considering these trade-offs, I'd like to propose an option to allow
> > superusers to disable HOT on tables. The intent is to trade some
> > performance benefits for the ability to reduce the size of a table
> > without the typical locking associated with it.
>
> Interesting use case, but I think that disabling HOT would be missing
> the forest for the trees. I think that a feature that disables
> block-local updates for pages > some offset would be a better solution
> to your issue: Normal updates also prefer the new tuple to be stored
> in the same pages as the old tuple if at all possible, so disabling
> HOT wouldn't solve the issue of tuples residing in the tail of your
> table - at least not while there is still empty space in those pages.
Hmm... I see your point. It's when an UPDATE isn't going to land on
the same page that it relocates to the earlier available page. So I
guess I'm after whatever mechanism would allow that to happen reliably
and predictably.
So $subject should really be "Allow forcing UPDATEs off the same page".
Thom
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: Disabling Heap-Only Tuples
2023-07-05 10:44 Disabling Heap-Only Tuples Thom Brown <[email protected]>
2023-07-05 10:57 ` Re: Disabling Heap-Only Tuples Matthias van de Meent <[email protected]>
2023-07-05 11:02 ` Re: Disabling Heap-Only Tuples Thom Brown <[email protected]>
@ 2023-07-05 12:12 ` Matthias van de Meent <[email protected]>
1 sibling, 0 replies; 6+ messages in thread
From: Matthias van de Meent @ 2023-07-05 12:12 UTC (permalink / raw)
To: Thom Brown <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Wed, 5 Jul 2023 at 13:03, Thom Brown <[email protected]> wrote:
>
> On Wed, 5 Jul 2023 at 11:57, Matthias van de Meent
> <[email protected]> wrote:
> >
> > On Wed, 5 Jul 2023 at 12:45, Thom Brown <[email protected]> wrote:
> > > Heap-Only Tuple (HOT) updates are a significant performance
> > > enhancement, as they prevent unnecessary page writes. However, HOT
> > > comes with a caveat: it means that if we have lots of available space
> > > earlier on in the relation, it can only be used for new tuples or in
> > > cases where there's insufficient space on a page for an UPDATE to use
> > > HOT.
> > >
> > > This mechanism limits our options for condensing tables, forcing us to
> > > resort to methods like running VACUUM FULL/CLUSTER or using external
> > > tools like pg_repack. These either require exclusive locks (which will
> > > be a deal-breaker on large tables on a production system), or there's
> > > risks involved. Of course we can always flood pages with new versions
> > > of a row until it's forced onto an early page, but that shouldn't be
> > > necessary.
> > >
> > > Considering these trade-offs, I'd like to propose an option to allow
> > > superusers to disable HOT on tables. The intent is to trade some
> > > performance benefits for the ability to reduce the size of a table
> > > without the typical locking associated with it.
> >
> > Interesting use case, but I think that disabling HOT would be missing
> > the forest for the trees. I think that a feature that disables
> > block-local updates for pages > some offset would be a better solution
> > to your issue: Normal updates also prefer the new tuple to be stored
> > in the same pages as the old tuple if at all possible, so disabling
> > HOT wouldn't solve the issue of tuples residing in the tail of your
> > table - at least not while there is still empty space in those pages.
>
> Hmm... I see your point. It's when an UPDATE isn't going to land on
> the same page that it relocates to the earlier available page. So I
> guess I'm after whatever mechanism would allow that to happen reliably
> and predictably.
>
> So $subject should really be "Allow forcing UPDATEs off the same page".
You'd probably want to do that only for a certain range of the table -
for a table with 1GB of data and 3GB of bloat there is no good reason
to force page-crossing updates in the first 1GB of the table - all
tuples of the table will eventually reside there, so why would you
take a performance penalty and move the tuples from inside that range
to inside that same range?
Something else to note: Indexes would suffer some (large?) amount of
bloat in this process, as you would be updating a lot of tuples
without the HOT optimization, thus increasing the work to be done by
VACUUM.
This may result in more bloat in indexes than what you get back from
shrinking the table.
Kind regards,
Matthias van de Meent
Neon (https://neon.tech/)
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: Disabling Heap-Only Tuples
2023-07-05 10:44 Disabling Heap-Only Tuples Thom Brown <[email protected]>
2023-07-05 10:57 ` Re: Disabling Heap-Only Tuples Matthias van de Meent <[email protected]>
2023-07-05 11:02 ` Re: Disabling Heap-Only Tuples Thom Brown <[email protected]>
@ 2023-07-05 19:47 ` Laurenz Albe <[email protected]>
1 sibling, 0 replies; 6+ messages in thread
From: Laurenz Albe @ 2023-07-05 19:47 UTC (permalink / raw)
To: Thom Brown <[email protected]>; Matthias van de Meent <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Wed, 2023-07-05 at 12:02 +0100, Thom Brown wrote:
> On Wed, 5 Jul 2023 at 11:57, Matthias van de Meent <[email protected]> wrote:
> > On Wed, 5 Jul 2023 at 12:45, Thom Brown <[email protected]> wrote:
> > > Heap-Only Tuple (HOT) updates are a significant performance
> > > enhancement, as they prevent unnecessary page writes. However, HOT
> > > comes with a caveat: it means that if we have lots of available space
> > > earlier on in the relation, it can only be used for new tuples or in
> > > cases where there's insufficient space on a page for an UPDATE to use
> > > HOT.
> > >
> > > Considering these trade-offs, I'd like to propose an option to allow
> > > superusers to disable HOT on tables. The intent is to trade some
> > > performance benefits for the ability to reduce the size of a table
> > > without the typical locking associated with it.
> >
> > Interesting use case, but I think that disabling HOT would be missing
> > the forest for the trees. I think that a feature that disables
> > block-local updates for pages > some offset would be a better solution
> > to your issue: Normal updates also prefer the new tuple to be stored
> > in the same pages as the old tuple if at all possible, so disabling
> > HOT wouldn't solve the issue of tuples residing in the tail of your
> > table - at least not while there is still empty space in those pages.
>
> Hmm... I see your point. It's when an UPDATE isn't going to land on
> the same page that it relocates to the earlier available page. So I
> guess I'm after whatever mechanism would allow that to happen reliably
> and predictably.
>
> So $subject should really be "Allow forcing UPDATEs off the same page".
I've been thinking about the same thing - an option that changes the update
strategy to always use the lowest block with enough free space.
That would allow to consolidate bloated tables with no down time.
Yours,
Laurenz Albe
^ permalink raw reply [nested|flat] 6+ messages in thread
end of thread, other threads:[~2023-07-05 19:47 UTC | newest]
Thread overview: 6+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2023-07-05 10:44 Disabling Heap-Only Tuples Thom Brown <[email protected]>
2023-07-05 10:57 ` Re: Disabling Heap-Only Tuples Matthias van de Meent <[email protected]>
2023-07-05 11:02 ` Re: Disabling Heap-Only Tuples Thom Brown <[email protected]>
2023-07-05 12:12 ` Re: Disabling Heap-Only Tuples Matthias van de Meent <[email protected]>
2023-07-05 19:47 ` Re: Disabling Heap-Only Tuples Laurenz Albe <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox