[PATCH v24 2/5] Add conditional lock feature to dshash

agora inbox for [email protected]  
help / color / mirror / Atom feed

[PATCH v24 2/5] Add conditional lock feature to dshash
230+ messages / 3 participants
[nested] [flat]

* [PATCH 2/5] Add conditional lock feature to dshash
@ 2018-09-27 02:15 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2018-09-27 02:15 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. This commit adds new
interfaces for dshash_find and dshash_find_or_insert. The new
interfaces have an extra parameter "nowait" taht commands not to wait
for lock.
---
 src/backend/lib/dshash.c | 64 +++++++++++++++++++++++++++++++++++++---
 src/include/lib/dshash.h |  6 ++++
 2 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 7675370058..7766fa7704 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -394,19 +394,51 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  */
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
+{
+	return dshash_find_extended(hash_table, key, exclusive, false, NULL);
+}
+
+/*
+ * The version of dshash_find, which is allowed to return immediately on lock
+ * failure. Lock status is set to *lock_failed in that case.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool *lock_acquired)
 {
 	dshash_hash hash;
 	size_t		partition;
 	dshash_table_item *item;
 
+	/*
+	 * No need to return lock resut when !nowait. Otherwise the caller may
+	 * omit the lock result when NULL is returned.
+	 */
+	Assert(nowait || !lock_acquired);
+
 	hash = hash_key(hash_table, key);
 	partition = PARTITION_FOR_HASH(hash);
 
 	Assert(hash_table->control->magic == DSHASH_MAGIC);
 	Assert(!hash_table->find_locked);
 
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
+	if (nowait)
+	{
+		if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partition),
+									  exclusive ? LW_EXCLUSIVE : LW_SHARED))
+		{
+			if (lock_acquired)
+				*lock_acquired = false;
+			return NULL;
+		}
+	}
+	else
+		LWLockAcquire(PARTITION_LOCK(hash_table, partition),
+					  exclusive ? LW_EXCLUSIVE : LW_SHARED);
+
+	if (lock_acquired)
+		*lock_acquired = true;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
@@ -441,6 +473,22 @@ void *
 dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
+{
+	return dshash_find_or_insert_extended(hash_table, key, found, false);
+}
+
+/*
+ * The version of dshash_find_or_insert, which is allowed to return immediately
+ * on lock failure.
+ *
+ * Notes above dshash_find_extended() regarding locking and error handling
+ * equally apply here.
+ */
+void *
+dshash_find_or_insert_extended(dshash_table *hash_table,
+							   const void *key,
+							   bool *found,
+							   bool nowait)
 {
 	dshash_hash hash;
 	size_t		partition_index;
@@ -455,8 +503,16 @@ dshash_find_or_insert(dshash_table *hash_table,
 	Assert(!hash_table->find_locked);
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (nowait)
+	{
+		if (!LWLockConditionalAcquire(
+				PARTITION_LOCK(hash_table, partition_index),
+				LW_EXCLUSIVE))
+			return NULL;
+	}
+	else
+		LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
+					  LW_EXCLUSIVE);
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index ad88f32cdd..a7d19c6a85 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -89,8 +89,14 @@ extern void dshash_destroy(dshash_table *hash_table);
 /* Finding, creating, deleting entries. */
 extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait,
+								  bool *lock_acquired);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_or_insert_extended(dshash_table *hash_table,
+											const void *key, bool *found,
+											bool nowait);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.23.0


----Next_Part(Wed_Jan_22_17_24_04_2020_620)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v24 2/5] Add conditional lock feature to dshash
@ 2018-09-27 02:15 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2018-09-27 02:15 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. This commit adds new
interfaces for dshash_find and dshash_find_or_insert. The new
interfaces have an extra parameter "nowait" taht commands not to wait
for lock.
---
 src/backend/lib/dshash.c | 69 ++++++++++++++++++++++++++++++++++++----
 src/include/lib/dshash.h |  6 ++++
 2 files changed, 68 insertions(+), 7 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 4f0c7ec840..60a6e3c0bc 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -394,19 +394,48 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  */
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
+{
+	return dshash_find_extended(hash_table, key, exclusive, false, NULL);
+}
+
+/*
+ * Addition to dshash_find, returns immediately when nowait is true and lock
+ * was not acquired. Lock status is set to *lock_failed if any.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool *lock_acquired)
 {
 	dshash_hash hash;
 	size_t		partition;
 	dshash_table_item *item;
 
+	/* allowing !nowait returning the result is just not sensible */
+	Assert(nowait || !lock_acquired);
+
 	hash = hash_key(hash_table, key);
 	partition = PARTITION_FOR_HASH(hash);
 
 	Assert(hash_table->control->magic == DSHASH_MAGIC);
 	Assert(!hash_table->find_locked);
 
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
+	if (nowait)
+	{
+		if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partition),
+									  exclusive ? LW_EXCLUSIVE : LW_SHARED))
+		{
+			if (lock_acquired)
+				*lock_acquired = false;
+			return NULL;
+		}
+	}
+	else
+		LWLockAcquire(PARTITION_LOCK(hash_table, partition),
+					  exclusive ? LW_EXCLUSIVE : LW_SHARED);
+
+	if (lock_acquired)
+		*lock_acquired = true;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
@@ -441,6 +470,22 @@ void *
 dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
+{
+	return dshash_find_or_insert_extended(hash_table, key, found, false);
+}
+
+/*
+ * Addition to dshash_find_or_insert, returns NULL if nowait is true and lock
+ * was not acquired.
+ *
+ * Notes above dshash_find_extended() regarding locking and error handling
+ * equally apply here.
+ */
+void *
+dshash_find_or_insert_extended(dshash_table *hash_table,
+							   const void *key,
+							   bool *found,
+							   bool nowait)
 {
 	dshash_hash hash;
 	size_t		partition_index;
@@ -455,8 +500,16 @@ dshash_find_or_insert(dshash_table *hash_table,
 	Assert(!hash_table->find_locked);
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (nowait)
+	{
+		if (!LWLockConditionalAcquire(
+				PARTITION_LOCK(hash_table, partition_index),
+				LW_EXCLUSIVE))
+			return NULL;
+	}
+	else
+		LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
+					  LW_EXCLUSIVE);
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
@@ -626,9 +679,11 @@ dshash_memhash(const void *v, size_t size, void *arg)
  * As opposed to the equivalent for dynanash, the caller is not supposed to
  * delete the returned element before continuing the scan.
  *
- * If consistent is set for dshash_seq_init, the whole hash table is
- * non-exclusively locked. Otherwise a part of the hash table is locked in the
- * same mode (partition lock).
+ * If consistent is set for dshash_seq_init, the all hash table
+ * partitions are locked in the requested mode (as determined by the
+ * exclusive flag), and the locks are held until the end of the scan.
+ * Otherwise the partition locks are acquired and released as needed
+ * during the scan (up to two partitions may be locked at the same time).
  */
 void
 dshash_seq_init(dshash_seq_status *status, dshash_table *hash_table,
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 79698a6ad6..67f7d77f71 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -90,8 +90,14 @@ extern void dshash_destroy(dshash_table *hash_table);
 /* Finding, creating, deleting entries. */
 extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait,
+								  bool *lock_acquired);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_or_insert_extended(dshash_table *hash_table,
+											const void *key, bool *found,
+											bool nowait);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.23.0


----Next_Part(Tue_Dec__3_17_27_59_2019_887)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v24-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v43 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index b829167872..9c90096f3d 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index c337099061..493e974832 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Dec_11_16_50_03_2020_915)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v43-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v33 3/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 1ef093e2e9..d7ee6de11e 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 0ca9514021..5f7a60febd 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.2


----Next_Part(Fri_May_15_17_30_36_2020_111)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v33-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v36 3/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 1ef093e2e9..d7ee6de11e 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 0ca9514021..5f7a60febd 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.4


----Next_Part(Tue_Sep__8_17_55_57_2020_197)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v36-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v35 3/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 1ef093e2e9..d7ee6de11e 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 0ca9514021..5f7a60febd 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.2


----Next_Part(Mon_Jun__8_17_32_04_2020_489)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v35-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v57 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Mar_18_16_56_02_2021_947)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v57-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v38 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index b829167872..9c90096f3d 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index c337099061..493e974832 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.4


----Next_Part(Thu_Oct__1_09_07_22_2020_252)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v38-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v40 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index b829167872..9c90096f3d 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index c337099061..493e974832 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.4


----Next_Part(Wed_Nov__4_17_39_10_2020_208)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v40-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v29 3/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 99 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 56 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index fb7e23c4cb..b4dc8e1ece 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,61 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "insert" indicates insert mode. In this mode new entry is inserted and set
+ * *found to false. *found is set to true if found. "found" must be non-null in
+ * this mode.
+ *
+ * If nowait is true, the function immediately returns if required lock was not
+ * acquired.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +484,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +499,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 81a929b8d9..80a896a99b 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.2


----Next_Part(Wed_Apr__1_15_15_11_2020_923)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v29-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v57 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Mar_18_16_56_02_2021_947)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v57-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v33 3/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 1ef093e2e9..d7ee6de11e 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 0ca9514021..5f7a60febd 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.2


----Next_Part(Thu_Apr_30_13_22_05_2020_409)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v33-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v32 3/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 1ef093e2e9..d7ee6de11e 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 0ca9514021..5f7a60febd 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.2


----Next_Part(Tue_Apr__7_16_38_17_2020_299)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v32-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v31 3/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 1ef093e2e9..d7ee6de11e 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 0ca9514021..5f7a60febd 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.2


----Next_Part(Fri_Apr__3_17_31_17_2020_104)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v31-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v37 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 1ef093e2e9..d7ee6de11e 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 0ca9514021..5f7a60febd 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.4


----Next_Part(Fri_Sep_25_09_27_26_2020_419)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v37-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v42 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index b829167872..9c90096f3d 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index c337099061..493e974832 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.4


----Next_Part(Wed_Nov_11_10_07_22_2020_796)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v42-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v39 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index b829167872..9c90096f3d 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index c337099061..493e974832 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.4


----Next_Part(Tue_Oct__6_10_06_44_2020_382)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v39-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v41 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index b829167872..9c90096f3d 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index c337099061..493e974832 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.4


----Next_Part(Fri_Nov__6_09_27_56_2020_738)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v41-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v34 3/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 1ef093e2e9..d7ee6de11e 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 0ca9514021..5f7a60febd 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.2


----Next_Part(Mon_Jun__1_18_00_01_2020_089)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v34-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v44 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index b829167872..9c90096f3d 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index c337099061..493e974832 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Mon_Dec_21_17_16_20_2020_608)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v44-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v25 3/8] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 117 ++++++++++++++++++++++++---------------
 src/include/lib/dshash.h |   3 +
 2 files changed, 75 insertions(+), 45 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 2086bdbea9..9a9b818d86 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -386,6 +386,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -395,36 +399,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -442,30 +417,61 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
-	dshash_table_item *item;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
 
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+/*
+ * Find the key in the hash table.
+ *
+ * "insert" indicates insert mode. In this mode new entry is inserted and set
+ * *found to false. *found is set to true if found. "found" must be non-null in
+ * this mode.  exclusive may be false in insert mode, but this function may
+ * take exclusive lock temporarily when actual insertion happens.
+ *
+ * If nowait is true, the function immediately returns if required lock was not
+ * acquired.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
+	dshash_table_item *item;
+	bool		inserted = false;
+
+	/* must be exclusive when insert allowed */
+	Assert(!insert || found != NULL);
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = !inserted;
+	}
 	else
 	{
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
+
 		*found = false;
 
 		/* Check if we are getting too full. */
@@ -482,26 +488,47 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
 		}
 
+		/* need to upgrade the lock to exclusive mode */
+		if (!exclusive)
+		{
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			LWLockAcquire(PARTITION_LOCK(hash_table, partidx), LW_EXCLUSIVE);
+		}
+
 		/* Finally we can try to insert the new item. */
 		item = insert_into_bucket(hash_table, key,
 								  &BUCKET_FOR_HASH(hash_table, hash));
 		item->hash = hash;
 		/* Adjust per-lock-partition counter for load factor knowledge. */
 		++partition->count;
+
+		if (!exclusive)
+		{
+			/*
+			 * The entry can be removed while downgrading lock. Re-find it for
+			 * safety.
+			 */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			inserted = true;
+
+			goto restart;
+		}
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 81a929b8d9..80a896a99b 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.2


----Next_Part(Thu_Mar_19_20_30_04_2020_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v25-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v30 3/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 1ef093e2e9..ee612b778f 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" means the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index 0ca9514021..5f7a60febd 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.18.2


----Next_Part(Wed_Apr__1_17_37_23_2020_570)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v30-0004-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v50 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_18_29_34_2021_806)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v50-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v48 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Mar__5_17_18_56_2021_497)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v48-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v45 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Fri_Jan__8_10_24_34_2021_185)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v45-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v51 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_12_10_39_2021_432)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v51-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v52 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Wed_Mar_10_17_51_37_2021_192)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v52-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v46 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_14_15_14_25_2021_903)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v46-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v49 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v47 2/7] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 520bfa0979..853d78b528 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Thu_Jan_21_12_03_48_2021_284)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v47-0003-Make-archiver-process-an-auxiliary-process.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH v56 2/6] Add conditional lock feature to dshash
@ 2020-03-13 07:58 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:58 UTC (permalink / raw)

Dshash currently waits for lock unconditionally. It is inconvenient
when we want to avoid being blocked by other processes. This commit
adds alternative functions of dshash_find and dshash_find_or_insert
that allows immediate return on lock failure.
---
 src/backend/lib/dshash.c | 98 +++++++++++++++++++++-------------------
 src/include/lib/dshash.h |  3 ++
 2 files changed, 55 insertions(+), 46 deletions(-)

diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 29ad767618..f79b6de245 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -383,6 +383,10 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
  * the caller must take care to ensure that the entry is not left corrupted.
  * The lock mode is either shared or exclusive depending on 'exclusive'.
  *
+ * If found is not NULL, *found is set to true if the key is found in the hash
+ * table. If the key is not found, *found is set to false and a pointer to a
+ * newly created entry is returned.
+ *
  * The caller must not lock a lock already.
  *
  * Note that the lock held is in fact an LWLock, so interrupts will be held on
@@ -392,36 +396,7 @@ dshash_get_hash_table_handle(dshash_table *hash_table)
 void *
 dshash_find(dshash_table *hash_table, const void *key, bool exclusive)
 {
-	dshash_hash hash;
-	size_t		partition;
-	dshash_table_item *item;
-
-	hash = hash_key(hash_table, key);
-	partition = PARTITION_FOR_HASH(hash);
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
-
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition),
-				  exclusive ? LW_EXCLUSIVE : LW_SHARED);
-	ensure_valid_bucket_pointers(hash_table);
-
-	/* Search the active bucket. */
-	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
-
-	if (!item)
-	{
-		/* Not found. */
-		LWLockRelease(PARTITION_LOCK(hash_table, partition));
-		return NULL;
-	}
-	else
-	{
-		/* The caller will free the lock by calling dshash_release_lock. */
-		hash_table->find_locked = true;
-		hash_table->find_exclusively_locked = exclusive;
-		return ENTRY_FROM_ITEM(item);
-	}
+	return dshash_find_extended(hash_table, key, exclusive, false, false, NULL);
 }
 
 /*
@@ -439,31 +414,60 @@ dshash_find_or_insert(dshash_table *hash_table,
 					  const void *key,
 					  bool *found)
 {
-	dshash_hash hash;
-	size_t		partition_index;
-	dshash_partition *partition;
+	return dshash_find_extended(hash_table, key, true, false, true, found);
+}
+
+
+/*
+ * Find the key in the hash table.
+ *
+ * "exclusive" is the lock mode in which the partition for the returned item
+ * is locked.  If "nowait" is true, the function immediately returns if
+ * required lock was not acquired.  "insert" indicates insert mode. In this
+ * mode new entry is inserted and set *found to false. *found is set to true if
+ * found. "found" must be non-null in this mode.
+ */
+void *
+dshash_find_extended(dshash_table *hash_table, const void *key,
+					 bool exclusive, bool nowait, bool insert, bool *found)
+{
+	dshash_hash hash = hash_key(hash_table, key);
+	size_t		partidx = PARTITION_FOR_HASH(hash);
+	dshash_partition *partition = &hash_table->control->partitions[partidx];
+	LWLockMode  lockmode = exclusive ? LW_EXCLUSIVE : LW_SHARED;
 	dshash_table_item *item;
 
-	hash = hash_key(hash_table, key);
-	partition_index = PARTITION_FOR_HASH(hash);
-	partition = &hash_table->control->partitions[partition_index];
-
-	Assert(hash_table->control->magic == DSHASH_MAGIC);
-	Assert(!hash_table->find_locked);
+	/* must be exclusive when insert allowed */
+	Assert(!insert || (exclusive && found != NULL));
 
 restart:
-	LWLockAcquire(PARTITION_LOCK(hash_table, partition_index),
-				  LW_EXCLUSIVE);
+	if (!nowait)
+		LWLockAcquire(PARTITION_LOCK(hash_table, partidx), lockmode);
+	else if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partidx),
+									   lockmode))
+		return NULL;
+
 	ensure_valid_bucket_pointers(hash_table);
 
 	/* Search the active bucket. */
 	item = find_in_bucket(hash_table, key, BUCKET_FOR_HASH(hash_table, hash));
 
 	if (item)
-		*found = true;
+	{
+		if (found)
+			*found = true;
+	}
 	else
 	{
-		*found = false;
+		if (found)
+			*found = false;
+
+		if (!insert)
+		{
+			/* The caller didn't told to add a new entry. */
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+			return NULL;
+		}
 
 		/* Check if we are getting too full. */
 		if (partition->count > MAX_COUNT_PER_PARTITION(hash_table))
@@ -479,7 +483,8 @@ restart:
 			 * Give up our existing lock first, because resizing needs to
 			 * reacquire all the locks in the right order to avoid deadlocks.
 			 */
-			LWLockRelease(PARTITION_LOCK(hash_table, partition_index));
+			LWLockRelease(PARTITION_LOCK(hash_table, partidx));
+
 			resize(hash_table, hash_table->size_log2 + 1);
 
 			goto restart;
@@ -493,12 +498,13 @@ restart:
 		++partition->count;
 	}
 
-	/* The caller must release the lock with dshash_release_lock. */
+	/* The caller will free the lock by calling dshash_release_lock. */
 	hash_table->find_locked = true;
-	hash_table->find_exclusively_locked = true;
+	hash_table->find_exclusively_locked = exclusive;
 	return ENTRY_FROM_ITEM(item);
 }
 
+
 /*
  * Remove an entry by key.  Returns true if the key was found and the
  * corresponding entry was removed.
diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h
index a6ea377173..5b8114d041 100644
--- a/src/include/lib/dshash.h
+++ b/src/include/lib/dshash.h
@@ -91,6 +91,9 @@ extern void *dshash_find(dshash_table *hash_table,
 						 const void *key, bool exclusive);
 extern void *dshash_find_or_insert(dshash_table *hash_table,
 								   const void *key, bool *found);
+extern void *dshash_find_extended(dshash_table *hash_table, const void *key,
+								  bool exclusive, bool nowait, bool insert,
+								  bool *found);
 extern bool dshash_delete_key(dshash_table *hash_table, const void *key);
 extern void dshash_delete_entry(dshash_table *hash_table, void *entry);
 extern void dshash_release_lock(dshash_table *hash_table, void *entry);
-- 
2.27.0


----Next_Part(Tue_Mar_16_10_27_55_2021_500)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v56-0003-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

* [PATCH 1/2] wip: add DYNAMIC_DEFAULT for settings which are vary by configure flags or platform or initdb
@ 2022-07-19 13:31 Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 230+ messages in thread

From: Justin Pryzby @ 2022-07-19 13:31 UTC (permalink / raw)

---
 doc/src/sgml/func.sgml       |  4 ++++
 src/backend/utils/misc/guc.c | 36 +++++++++++++++++++++++-------------
 src/include/utils/guc.h      |  6 ++++--
 3 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index e3e24e185a4..0fd6234eaf2 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -24977,6 +24977,10 @@ SELECT currval(pg_get_serial_sequence('sometable', 'id'));
         an empty array if the GUC exists but there are no flags to show.
         Only the most useful flags are exposed, as of the following:
         <simplelist>
+         <member>
+          <literal>DYNAMIC_DEFAULT</literal>: parameters whose default varies by
+          platform, or compile-time options, or initdb.
+         </member>
          <member>
           <literal>EXPLAIN</literal>: parameters included in
           <command>EXPLAIN (SETTINGS)</command> commands.
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0328029d430..dbbadc5a475 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -1435,7 +1435,7 @@ static struct config_bool ConfigureNamesBool[] =
 		{"debug_assertions", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows whether the running server has assertion checks enabled."),
 			NULL,
-			GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE
+			GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE | GUC_DYNAMIC_DEFAULT
 		},
 		&assert_enabled,
 #ifdef USE_ASSERT_CHECKING
@@ -1611,7 +1611,8 @@ static struct config_bool ConfigureNamesBool[] =
 	{
 		{"update_process_title", PGC_SUSET, PROCESS_TITLE,
 			gettext_noop("Updates the process title to show the active SQL command."),
-			gettext_noop("Enables updating of the process title every time a new SQL command is received by the server.")
+			gettext_noop("Enables updating of the process title every time a new SQL command is received by the server."),
+			GUC_DYNAMIC_DEFAULT
 		},
 		&update_process_title,
 #ifdef WIN32
@@ -2362,6 +2363,7 @@ static struct config_int ConfigureNamesInt[] =
 			gettext_noop("Sets the maximum number of concurrent connections."),
 			NULL
 		},
+
 		&MaxConnections,
 		100, 1, MAX_BACKENDS,
 		check_maxconnections, NULL, NULL
@@ -2397,7 +2399,7 @@ static struct config_int ConfigureNamesInt[] =
 		{"shared_buffers", PGC_POSTMASTER, RESOURCES_MEM,
 			gettext_noop("Sets the number of shared memory buffers used by the server."),
 			NULL,
-			GUC_UNIT_BLOCKS
+			GUC_UNIT_BLOCKS | GUC_DYNAMIC_DEFAULT
 		},
 		&NBuffers,
 		16384, 16, INT_MAX / 2,
@@ -3142,7 +3144,7 @@ static struct config_int ConfigureNamesInt[] =
 			RESOURCES_ASYNCHRONOUS,
 			gettext_noop("Number of simultaneous requests that can be handled efficiently by the disk subsystem."),
 			NULL,
-			GUC_EXPLAIN
+			GUC_EXPLAIN|GUC_DYNAMIC_DEFAULT
 		},
 		&effective_io_concurrency,
 #ifdef USE_PREFETCH
@@ -3160,7 +3162,7 @@ static struct config_int ConfigureNamesInt[] =
 			RESOURCES_ASYNCHRONOUS,
 			gettext_noop("A variant of effective_io_concurrency that is used for maintenance work."),
 			NULL,
-			GUC_EXPLAIN
+			GUC_EXPLAIN|GUC_DYNAMIC_DEFAULT
 		},
 		&maintenance_io_concurrency,
 #ifdef USE_PREFETCH
@@ -3327,9 +3329,11 @@ static struct config_int ConfigureNamesInt[] =
 			gettext_noop("Shows the size of write ahead log segments."),
 			NULL,
 			GUC_UNIT_BYTE | GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE | GUC_RUNTIME_COMPUTED
+
 		},
 		&wal_segment_size,
 		DEFAULT_XLOG_SEG_SIZE,
+
 		WalSegMinSize,
 		WalSegMaxSize,
 		NULL, NULL, NULL
@@ -3523,6 +3527,7 @@ static struct config_int ConfigureNamesInt[] =
 			GUC_UNIT_BLOCKS | GUC_EXPLAIN,
 		},
 		&effective_cache_size,
+
 		DEFAULT_EFFECTIVE_CACHE_SIZE, 1, INT_MAX,
 		NULL, NULL, NULL
 	},
@@ -3620,7 +3625,7 @@ static struct config_int ConfigureNamesInt[] =
 		{"debug_discard_caches", PGC_SUSET, DEVELOPER_OPTIONS,
 			gettext_noop("Aggressively flush system caches for debugging purposes."),
 			NULL,
-			GUC_NOT_IN_SAMPLE
+			GUC_NOT_IN_SAMPLE | GUC_DYNAMIC_DEFAULT
 		},
 		&debug_discard_caches,
 #ifdef DISCARD_CACHES_ENABLED
@@ -4447,7 +4452,7 @@ static struct config_string ConfigureNamesString[] =
 		{"unix_socket_directories", PGC_POSTMASTER, CONN_AUTH_SETTINGS,
 			gettext_noop("Sets the directories where Unix-domain sockets will be created."),
 			NULL,
-			GUC_LIST_INPUT | GUC_LIST_QUOTE | GUC_SUPERUSER_ONLY
+			GUC_LIST_INPUT | GUC_LIST_QUOTE | GUC_SUPERUSER_ONLY | GUC_DYNAMIC_DEFAULT
 		},
 		&Unix_socket_directories,
 #ifdef HAVE_UNIX_SOCKETS
@@ -4532,7 +4537,7 @@ static struct config_string ConfigureNamesString[] =
 		{"ssl_library", PGC_INTERNAL, PRESET_OPTIONS,
 			gettext_noop("Shows the name of the SSL library."),
 			NULL,
-			GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE
+			GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE | GUC_DYNAMIC_DEFAULT
 		},
 		&ssl_library,
 #ifdef USE_SSL
@@ -4618,7 +4623,7 @@ static struct config_string ConfigureNamesString[] =
 		{"ssl_ciphers", PGC_SIGHUP, CONN_AUTH_SSL,
 			gettext_noop("Sets the list of allowed SSL ciphers."),
 			NULL,
-			GUC_SUPERUSER_ONLY
+			GUC_SUPERUSER_ONLY | GUC_DYNAMIC_DEFAULT
 		},
 		&SSLCipherSuites,
 #ifdef USE_OPENSSL
@@ -4633,7 +4638,7 @@ static struct config_string ConfigureNamesString[] =
 		{"ssl_ecdh_curve", PGC_SIGHUP, CONN_AUTH_SSL,
 			gettext_noop("Sets the curve to use for ECDH."),
 			NULL,
-			GUC_SUPERUSER_ONLY
+			GUC_SUPERUSER_ONLY | GUC_DYNAMIC_DEFAULT
 		},
 		&SSLECDHCurve,
 #ifdef USE_SSL
@@ -4871,7 +4876,8 @@ static struct config_enum ConfigureNamesEnum[] =
 	{
 		{"syslog_facility", PGC_SIGHUP, LOGGING_WHERE,
 			gettext_noop("Sets the syslog \"facility\" to be used when syslog enabled."),
-			NULL
+			NULL,
+			GUC_DYNAMIC_DEFAULT
 		},
 		&syslog_facility,
 #ifdef HAVE_SYSLOG
@@ -4984,7 +4990,8 @@ static struct config_enum ConfigureNamesEnum[] =
 	{
 		{"dynamic_shared_memory_type", PGC_POSTMASTER, RESOURCES_MEM,
 			gettext_noop("Selects the dynamic shared memory implementation used."),
-			NULL
+			NULL,
+			GUC_DYNAMIC_DEFAULT
 		},
 		&dynamic_shared_memory_type,
 		DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE, dynamic_shared_memory_options,
@@ -5008,6 +5015,7 @@ static struct config_enum ConfigureNamesEnum[] =
 		},
 		&sync_method,
 		DEFAULT_SYNC_METHOD, sync_method_options,
+
 		NULL, assign_xlog_sync_method, NULL
 	},
 
@@ -9943,7 +9951,7 @@ GetConfigOptionByName(const char *name, const char **varname, bool missing_ok)
 Datum
 pg_settings_get_flags(PG_FUNCTION_ARGS)
 {
-#define MAX_GUC_FLAGS	5
+#define MAX_GUC_FLAGS	6
 	char	   *varname = TextDatumGetCString(PG_GETARG_DATUM(0));
 	struct config_generic *record;
 	int			cnt = 0;
@@ -9956,6 +9964,8 @@ pg_settings_get_flags(PG_FUNCTION_ARGS)
 	if (record == NULL)
 		PG_RETURN_NULL();
 
+	if (record->flags & GUC_DYNAMIC_DEFAULT)
+		flags[cnt++] = CStringGetTextDatum("DYNAMIC_DEFAULT");
 	if (record->flags & GUC_EXPLAIN)
 		flags[cnt++] = CStringGetTextDatum("EXPLAIN");
 	if (record->flags & GUC_NO_RESET_ALL)
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index 4d0920c42e2..bb09c507f81 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -221,14 +221,14 @@ typedef enum
 
 #define GUC_UNIT_KB				0x1000	/* value is in kilobytes */
 #define GUC_UNIT_BLOCKS			0x2000	/* value is in blocks */
-#define GUC_UNIT_XBLOCKS		0x3000	/* value is in xlog blocks */
+#define GUC_UNIT_XBLOCKS		0x3000	/* value is in xlog blocks */ //
 #define GUC_UNIT_MB				0x4000	/* value is in megabytes */
 #define GUC_UNIT_BYTE			0x8000	/* value is in bytes */
 #define GUC_UNIT_MEMORY			0xF000	/* mask for size-related units */
 
 #define GUC_UNIT_MS			   0x10000	/* value is in milliseconds */
 #define GUC_UNIT_S			   0x20000	/* value is in seconds */
-#define GUC_UNIT_MIN		   0x30000	/* value is in minutes */
+#define GUC_UNIT_MIN		   0x30000	/* value is in minutes */ //
 #define GUC_UNIT_TIME		   0xF0000	/* mask for time-related units */
 
 #define GUC_EXPLAIN			  0x100000	/* include in explain */
@@ -239,6 +239,8 @@ typedef enum
  */
 #define GUC_RUNTIME_COMPUTED  0x200000
 
+#define GUC_DYNAMIC_DEFAULT		  0x400000	/* default value is dynamic */
+
 #define GUC_UNIT				(GUC_UNIT_MEMORY | GUC_UNIT_TIME)
 
 
-- 
2.17.1


--xaMk4Io5JJdpkLEb
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0002-Add-fileval-bootval-consistency-check-of-GUC-paramet.patch"



^ permalink  raw  reply  [nested|flat] 230+ messages in thread

end of thread, other threads:[~2022-07-19 13:31 UTC | newest]

Thread overview: 230+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2018-09-27 02:15 [PATCH 2/5] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2018-09-27 02:15 [PATCH v24 2/5] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v43 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v33 3/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v36 3/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v35 3/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v57 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v38 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v40 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v29 3/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v57 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v33 3/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v32 3/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v31 3/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v37 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v42 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v39 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v41 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v34 3/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v44 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v25 3/8] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v30 3/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v50 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v48 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v45 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v51 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v52 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v46 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v49 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v47 2/7] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2020-03-13 07:58 [PATCH v56 2/6] Add conditional lock feature to dshash Kyotaro Horiguchi <[email protected]>
2022-07-19 13:31 [PATCH 1/2] wip: add DYNAMIC_DEFAULT for settings which are vary by configure flags or platform or initdb Justin Pryzby <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox