public inbox for [email protected]
help / color / mirror / Atom feedRe: proposal: Support Unicode host variable in ECPG
27+ messages / 4 participants
[nested] [flat]
* Re: proposal: Support Unicode host variable in ECPG
@ 2017-10-26 03:29 Jing Wang <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Jing Wang @ 2017-10-26 03:29 UTC (permalink / raw)
To: pgsql-hackers
Hi,
I would like to provide the patch file that ECPG can support the
utf16/utf32 unicode host variable type. This feature is used to compatible
the same feature in Oracle's. Please find Oracle's feature in the [1].
[1] https://docs.oracle.com/database/121/LNPCC/pc_05adv.htm#LNPCC3273
Regards,
Jing Wang
Fujitsu Australia
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Attachments:
[application/octet-stream] ecpg_utext_v0.1.patch (700.3K, ../../CAF3+xM+Z+eyhTVr0PJwqBE9FAAG-1Wi6PeqJA02MHdfgB+_14w@mail.gmail.com/3-ecpg_utext_v0.1.patch)
download | inline diff:
diff --git a/src/backend/utils/mb/encnames.c b/src/backend/utils/mb/encnames.c
index 12b61cd..76e2e20 100644
--- a/src/backend/utils/mb/encnames.c
+++ b/src/backend/utils/mb/encnames.c
@@ -402,6 +402,53 @@ const pg_enc2gettext pg_enc2gettext_tbl[] =
{0, NULL}
};
+const pg_encmaxlen pg_encmaxlen_tbl[] =
+{
+ {PG_SQL_ASCII, 1},
+ {PG_EUC_JP, 1},
+ {PG_EUC_CN, 1},
+ {PG_EUC_KR, 1},
+ {PG_EUC_TW, 1},
+ {PG_EUC_JIS_2004, 1},
+ {PG_UTF8, 4},
+ {PG_MULE_INTERNAL, 1},
+ {PG_LATIN1, 1},
+ {PG_LATIN2, 1},
+ {PG_LATIN3, 1},
+ {PG_LATIN4, 1},
+ {PG_LATIN5, 1},
+ {PG_LATIN6, 1},
+ {PG_LATIN7, 1},
+ {PG_LATIN8, 1},
+ {PG_LATIN9, 1},
+ {PG_LATIN10, 1},
+ {PG_WIN1256, 1},
+ {PG_WIN1258, 1},
+ {PG_WIN866, 1},
+ {PG_WIN874, 1},
+ {PG_KOI8R, 1},
+ {PG_WIN1251, 1},
+ {PG_WIN1252, 1},
+ {PG_ISO_8859_5, 1},
+ {PG_ISO_8859_6, 1},
+ {PG_ISO_8859_7, 1},
+ {PG_ISO_8859_8, 1},
+ {PG_WIN1250, 1},
+ {PG_WIN1253, 1},
+ {PG_WIN1254, 1},
+ {PG_WIN1255, 1},
+ {PG_WIN1256, 1},
+ {PG_WIN1257, 1},
+ {PG_KOI8U, 1},
+ {PG_SJIS, 2},
+ {PG_BIG5, 2},
+ {PG_GBK, 2},
+ {PG_UHC, 2},
+ {PG_GB18030, 4},
+ {PG_JOHAB, 3},
+ {PG_SHIFT_JIS_2004, 2},
+ {0, 0}
+};
#ifndef FRONTEND
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index d57ef01..8f345ed 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -334,6 +334,14 @@ typedef struct pg_enc2gettext
extern const pg_enc2gettext pg_enc2gettext_tbl[];
+typedef struct pg_encmaxlen
+{
+ pg_enc encoding;
+ int len;
+} pg_encmaxlen;
+
+extern const pg_encmaxlen pg_encmaxlen_tbl[];
+
/*
* Encoding names for ICU
*/
diff --git a/src/interfaces/ecpg/ecpglib/Makefile b/src/interfaces/ecpg/ecpglib/Makefile
index fbb1407..0fd43d4 100644
--- a/src/interfaces/ecpg/ecpglib/Makefile
+++ b/src/interfaces/ecpg/ecpglib/Makefile
@@ -26,7 +26,7 @@ override CFLAGS += $(PTHREAD_CFLAGS)
LIBS := $(filter-out -lpgport, $(LIBS))
OBJS= execute.o typename.o descriptor.o sqlda.o data.o error.o prepare.o memory.o \
- connect.o misc.o path.o pgstrcasecmp.o \
+ connect.o misc.o path.o pgstrcasecmp.o convertutf.o \
$(filter snprintf.o strlcpy.o win32setlocale.o isinf.o, $(LIBOBJS)) $(WIN32RES)
# thread.c is needed only for non-WIN32 implementation of path.c
diff --git a/src/interfaces/ecpg/ecpglib/convertutf.c b/src/interfaces/ecpg/ecpglib/convertutf.c
new file mode 100644
index 0000000..03bb617
--- /dev/null
+++ b/src/interfaces/ecpg/ecpglib/convertutf.c
@@ -0,0 +1,180 @@
+/*
+/* src/interfaces/ecpg/ecpglib/convertutf.c */
+
+#include "convertutf.h"
+#include "c.h"
+#ifdef CVTUTF_DEBUG
+#include <stdio.h>
+#endif
+#include <iconv.h>
+#include <errno.h>
+
+#ifndef false
+#define false 0
+#endif
+
+#ifndef true
+#define true 1
+#endif
+
+static bool is_little_endian(void);
+
+/* Return true if utext string is null*/
+bool utext_is_null(char *source)
+{
+#if defined(WIN32)
+ if(*(short int*)source == 0)
+#else
+ if(*(int*)source == 0)
+#endif
+ return true;
+
+ return false;
+}
+
+
+/*****
+ * Get the UTF32 or UTF16 string length by characters.
+ * Four bytes UTF16 consider it as 2 characters.
+ * maxlen means the max character length of the unicode string.
+ * if maxlen = 0, then get the length until find the 0x0000.
+ */
+long get_utext_length(char *source, unsigned int maxlen)
+{
+ long len = 0;
+
+ while(1)
+ {
+#if defined(WIN32)
+ if ((short)(*source) == 0x00)
+#else
+ if ((int)(*source) == 0x00)
+#endif
+ break;
+
+ len++;
+ source += UNICODE_CH_LEN;
+
+ if(maxlen && len>=maxlen)
+ return maxlen;
+ }
+
+ return len;
+}
+
+ConversionResult
+convert_func(char *from_code, char *to_code, char *from, size_t from_len, char *to, size_t to_len, int *real_len)
+{
+ iconv_t cd;
+ int len = to_len;
+ ConversionResult ret = conversionOK;
+
+ //cd = iconv_open("UTF-16LE","SHIFT-JIS");
+ cd = iconv_open(to_code,from_code);
+
+ if((cd) == (iconv_t) -1)
+ return conversionUnsupported;
+
+ if(iconv(cd, &from, &from_len, &to, &to_len) == -1)
+ {
+ int num = 0;
+
+#if defined(WIN32)
+ num = GetLastError();
+ if (num ==0 && to_len == 0 && from_len > 0)
+ num = E2BIG;
+#else
+ num = errno;
+#endif
+
+ if(num == E2BIG)
+ ret = targetExhausted;
+ else
+ ret = sourceIllegal;
+ }
+ else
+ {
+ if(real_len)
+ *real_len = (len - to_len);
+ }
+
+ iconv_close(cd);
+
+ return ret;
+}
+
+/*
+ * This function return the need converted length by bytes.
+ */
+int get_converted_length(char *from_code, char *to_code, char *from, size_t from_len)
+{
+ iconv_t cd;
+ size_t to_len = from_len *4;
+ int max_len;
+ int convert_ret;
+ char buffer[512]={0};
+ char *to = buffer;
+ bool need_free = false;
+
+ if (to_len > 512)
+ {
+ to = (char*)malloc(to_len);
+ memset(to,0,to_len);
+ need_free = true;
+ }
+
+ max_len = to_len;
+
+ cd = iconv_open(to_code,from_code);
+ if((cd) == (iconv_t) -1)
+ return conversionUnsupported;
+
+ convert_ret = (int)iconv(cd, &from, &from_len, &to, &to_len);
+
+ iconv_close(cd);
+
+ if(need_free)
+ free(to);
+
+ if(convert_ret==-1)
+ return convert_ret;
+
+ return (max_len - to_len);
+}
+
+/*****
+ * Return the unicode encoding name based on the platform
+ */
+char *
+get_utext_encoding()
+{
+#if defined(WIN32)
+ if(is_little_endian())
+ return "UTF-16LE";
+ else
+ return "UTF-16BE";
+#else
+ if(is_little_endian())
+ return "UTF32LE";
+ else
+ return "UTF32BE";
+#endif
+}
+
+/*
+ * Check which type of endianness format supported by the machine
+ * Return:
+ * True -- little endian
+ * False -- big endian
+ *
+ */
+static bool
+is_little_endian()
+{
+ int n = 1;
+
+ if(*(char *)&n == 1)
+ return true;
+
+ return false;
+}
diff --git a/src/interfaces/ecpg/ecpglib/data.c b/src/interfaces/ecpg/ecpglib/data.c
index a2f3916..509329e 100644
--- a/src/interfaces/ecpg/ecpglib/data.c
+++ b/src/interfaces/ecpg/ecpglib/data.c
@@ -15,6 +15,7 @@
#include "pgtypes_date.h"
#include "pgtypes_timestamp.h"
#include "pgtypes_interval.h"
+#include "convertutf.h"
/* returns true if character c is a delimiter for the given array type */
static bool
@@ -235,6 +236,8 @@ ecpg_get_data(const PGresult *results, int act_tuple, int act_field, int lineno,
case ECPGt_unsigned_char:
case ECPGt_varchar:
case ECPGt_string:
+ case ECPGt_utext:
+ case ECPGt_uvarchar:
break;
default:
@@ -506,7 +509,110 @@ ecpg_get_data(const PGresult *results, int act_tuple, int act_field, int lineno,
pval += size;
}
break;
+ case ECPGt_utext:
+ {
+ int real_len = 0;
+ int utext_len = 0;
+ char *str = (char *) (var + offset * act_tuple);
+ char *encoding = NULL;
+ int pval_len = size;
+ int to_len;
+ char *dest_encoding = get_utext_encoding();
+ int dest_character_size = UNICODE_CH_LEN;
+
+ ConversionResult convert_ret;
+
+ /* Here will be modified later by setting length according to encoding */
+ to_len = varcharsize>0?offset:size*dest_character_size;
+
+ /* Check the PGCLIENTENCODING */
+ encoding = PQGetEncodingName(PQGetEncodingFromPGres(results));
+ if(encoding == NULL)
+ {
+ ecpg_raise(lineno, ECPG_CLIENT_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_INTERNAL_ERROR, NULL);
+ return false;
+ }
+
+ /*
+ * If varcharsize is unknown and the offset is that of
+ * char *, then this variable represents the array of
+ * character pointers. So, use extra indirection.
+ */
+ if (varcharsize == 0 && offset == sizeof(char *))
+ str = *(char **) str;
+
+ /*
+ * Don't know the real size of the utext pointer,
+ * so set the offset to be needed size.
+ * For utext pointer to pointer host variable with
+ * uninitialized the offset will be set zero as input
+ * parameter.
+ * For utext pointer host varaible the varcharsize is
+ * zero and offset is not.
+ */
+ if (varcharsize == 0 || offset == 0)
+ {
+ real_len = get_converted_length(encoding,dest_encoding,pval,pval_len);
+ to_len = offset = real_len;
+ memset(str,0,offset+dest_character_size);
+ }
+ else
+ memset(str,0,offset);
+
+ /* Start to convert encoding to UTF16 */
+ convert_ret = convert_func(encoding,dest_encoding, pval,pval_len,str,to_len,&real_len);
+ if (convert_ret == sourceIllegal || convert_ret == conversionUnsupported)
+ {
+ ecpg_raise(lineno, ECPG_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_ENCODING_ERROR, NULL);
+ return false;
+ }
+ else if (convert_ret == targetExhausted)
+ {
+ real_len = get_converted_length(encoding,dest_encoding,pval,pval_len);
+ if(real_len == -1)
+ {
+ ecpg_raise(lineno, ECPG_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_ENCODING_ERROR, NULL);
+ return false;
+ }
+ }
+
+ utext_len = real_len/dest_character_size;
+ /*
+ * If the need bytes by UTF16 string is larger than the
+ * length of host variable, set the ind variable
+ */
+ if (varcharsize > 0 && varcharsize < utext_len)
+ {
+ /* truncation */
+ switch (ind_type)
+ {
+ case ECPGt_short:
+ case ECPGt_unsigned_short:
+ *((short *) (ind + ind_offset * act_tuple)) = utext_len;
+ break;
+ case ECPGt_int:
+ case ECPGt_unsigned_int:
+ *((int *) (ind + ind_offset * act_tuple)) = utext_len;
+ break;
+ case ECPGt_long:
+ case ECPGt_unsigned_long:
+ *((long *) (ind + ind_offset * act_tuple)) = utext_len;
+ break;
+#ifdef HAVE_LONG_LONG_INT
+ case ECPGt_long_long:
+ case ECPGt_unsigned_long_long:
+ *((long long int *) (ind + ind_offset * act_tuple)) = utext_len;
+ break;
+#endif /* HAVE_LONG_LONG_INT */
+ default:
+ break;
+ }
+ sqlca->sqlwarn[0] = sqlca->sqlwarn[1] = 'W';
+ }
+ pval += size;
+ }
+ break;
case ECPGt_varchar:
{
struct ECPGgeneric_varchar *variable =
@@ -554,6 +660,87 @@ ecpg_get_data(const PGresult *results, int act_tuple, int act_field, int lineno,
}
break;
+ case ECPGt_uvarchar:
+ {
+ int real_len = 0;
+ int utext_len = 0;
+ int to_len = 0;
+ int pval_len = size;
+ char *encoding = NULL;
+ ConversionResult convert_ret = conversionOK;
+ struct ECPGgeneric_varchar *variable =
+ (struct ECPGgeneric_varchar *) (var + offset * act_tuple);
+
+ char *str = (char *) (variable->arr);
+
+ char *dest_encoding = get_utext_encoding();
+ int dest_character_size = UNICODE_CH_LEN;
+
+ memset((char*)variable, 0, offset);
+
+ /* Check the PGCLIENTENCODING */
+ encoding = PQGetEncodingName(PQGetEncodingFromPGres(results));
+ if(encoding == NULL)
+ {
+ ecpg_raise(lineno, ECPG_CLIENT_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_INTERNAL_ERROR, NULL);
+ return false;
+ }
+
+ to_len = offset-sizeof(int);
+
+ /* Start to convert encoding to UTF16 or UTF32 */
+ convert_ret = convert_func(encoding,dest_encoding, pval,pval_len,str,to_len,&real_len);
+
+ if (convert_ret == sourceIllegal || convert_ret == conversionUnsupported)
+ {
+ ecpg_raise(lineno, ECPG_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_ENCODING_ERROR, NULL);
+ return false;
+ }
+ else if (convert_ret == targetExhausted)
+ {
+ real_len = get_converted_length(encoding,dest_encoding,pval,pval_len);
+ if(real_len == -1)
+ {
+ ecpg_raise(lineno, ECPG_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_ENCODING_ERROR, NULL);
+ return false;
+ }
+ }
+
+ /* Get the actual length of UTF16 characters */
+ utext_len = real_len/dest_character_size;
+ variable->len = Min(utext_len,varcharsize);
+
+ if (utext_len > varcharsize)
+ {
+ /* truncation */
+ switch (ind_type)
+ {
+ case ECPGt_short:
+ case ECPGt_unsigned_short:
+ *((short *) (ind + ind_offset * act_tuple)) = utext_len;
+ break;
+ case ECPGt_int:
+ case ECPGt_unsigned_int:
+ *((int *) (ind + ind_offset * act_tuple)) = utext_len;
+ break;
+ case ECPGt_long:
+ case ECPGt_unsigned_long:
+ *((long *) (ind + ind_offset * act_tuple)) = utext_len;
+ break;
+#ifdef HAVE_LONG_LONG_INT
+ case ECPGt_long_long:
+ case ECPGt_unsigned_long_long:
+ *((long long int *) (ind + ind_offset * act_tuple)) = utext_len;
+ break;
+#endif /* HAVE_LONG_LONG_INT */
+ default:
+ break;
+ }
+ sqlca->sqlwarn[0] = sqlca->sqlwarn[1] = 'W';
+ }
+ pval += size;
+ }
+ break;
case ECPGt_decimal:
case ECPGt_numeric:
for (endptr = pval; *endptr && *endptr != ',' && *endptr != '}'; endptr++);
diff --git a/src/interfaces/ecpg/ecpglib/error.c b/src/interfaces/ecpg/ecpglib/error.c
index f34ae4a..80979d5 100644
--- a/src/interfaces/ecpg/ecpglib/error.c
+++ b/src/interfaces/ecpg/ecpglib/error.c
@@ -40,6 +40,16 @@ ecpg_raise(int line, int code, const char *sqlstate, const char *str)
ecpg_gettext("out of memory on line %d"), line);
break;
+ case ECPG_ENCODING_ERROR:
+ snprintf(sqlca->sqlerrm.sqlerrmc, sizeof(sqlca->sqlerrm.sqlerrmc),
+
+ /*
+ * translator: this string will be truncated at 149 characters
+ * expanded.
+ */
+ ecpg_gettext("encoding conversion error on line %d"), line);
+ break;
+
case ECPG_UNSUPPORTED:
snprintf(sqlca->sqlerrm.sqlerrmc, sizeof(sqlca->sqlerrm.sqlerrmc),
/*------
@@ -200,6 +210,15 @@ ecpg_raise(int line, int code, const char *sqlstate, const char *str)
ecpg_gettext("could not connect to database \"%s\" on line %d"), str, line);
break;
+ case ECPG_CLIENT_ENCODING_ERROR:
+ snprintf(sqlca->sqlerrm.sqlerrmc, sizeof(sqlca->sqlerrm.sqlerrmc),
+
+ /*
+ * translator: this string will be truncated at 149 characters
+ * expanded.
+ */
+ ecpg_gettext("Failed to get client encoding name on line %d"),line);
+ break;
default:
snprintf(sqlca->sqlerrm.sqlerrmc, sizeof(sqlca->sqlerrm.sqlerrmc),
/*------
diff --git a/src/interfaces/ecpg/ecpglib/execute.c b/src/interfaces/ecpg/ecpglib/execute.c
index 50f831a..1e331d0 100644
--- a/src/interfaces/ecpg/ecpglib/execute.c
+++ b/src/interfaces/ecpg/ecpglib/execute.c
@@ -33,7 +33,10 @@
#include "pgtypes_date.h"
#include "pgtypes_timestamp.h"
#include "pgtypes_interval.h"
+#include "convertutf.h"
+static int ecpg_do_utext_encoding(struct variable * var, int client_encoding,int lineno);
+static int ecpg_do_uvachar_encoding(struct variable * var, int client_encoding,int lineno);
/*
* This function returns a newly malloced string that has ' and \
* escaped.
@@ -88,6 +91,11 @@ free_variable(struct variable *var)
while (var)
{
+ if (var->alloc_inf == 1)
+ ecpg_free(var->value);
+ if (var->ind_alloc_inf == 1)
+ ecpg_free(var->ind_value);
+
var_next = var->next;
ecpg_free(var);
var = var_next;
@@ -382,9 +390,83 @@ ecpg_store_result(const PGresult *results, int act_field,
len = var->offset * ntuples;
}
break;
+ case ECPGt_utext:
+ {
+ char *pval;
+ int pval_len;
+ int unicode_str_len;
+
+ char *src_encoding;
+ char *dest_encoding = get_utext_encoding();
+
+ /* Check the PGCLIENTENCODING */
+ src_encoding = PQGetEncodingName(PQGetEncodingFromPGres(results));
+ if(src_encoding == NULL)
+ {
+ ecpg_raise(stmt->lineno, ECPG_CLIENT_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_INTERNAL_ERROR, NULL);
+ return false;
+ }
+
+ if (!var->varcharsize && !var->arrsize)
+ {
+ /* special mode for handling utext **foo=0 */
+ for (act_tuple = 0; act_tuple < ntuples; act_tuple++)
+ {
+ pval = (char *) PQgetvalue(results, act_tuple, act_field);
+ pval_len = PQgetlength(results, act_tuple, act_field);
+
+ unicode_str_len = get_converted_length(src_encoding,dest_encoding,pval,pval_len);
+ if(unicode_str_len == -1)
+ {
+ ecpg_raise(stmt->lineno, ECPG_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_ENCODING_ERROR, NULL);
+ return false;
+ }
+
+ len += unicode_str_len + UNICODE_CH_LEN;
+ }
+ len += (ntuples + 1) * sizeof(char *);
+ }
+ else
+ {
+ var->varcharsize = 0;
+ /* check strlen for each tuple */
+ for (act_tuple = 0; act_tuple < ntuples; act_tuple++)
+ {
+ pval = (char *) PQgetvalue(results, act_tuple, act_field);
+ pval_len = PQgetlength(results, act_tuple, act_field);
+
+ unicode_str_len = get_converted_length(src_encoding,dest_encoding,pval,pval_len);
+ if(unicode_str_len == -1)
+ {
+ ecpg_raise(stmt->lineno, ECPG_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_ENCODING_ERROR, NULL);
+ return false;
+ }
+
+ /* utf16_str_len indicates the utf16 string length by bytes */
+ unicode_str_len = unicode_str_len + UNICODE_CH_LEN;
+
+ if (unicode_str_len > var->varcharsize)
+ var->varcharsize = unicode_str_len;
+ }
+
+ /* Here offset is the utf16 length by bytes,
+ * so no need using '*' as used in char
+ */
+ var->offset = var->varcharsize;
+
+ /* varcharsize contains number of characters so divided by UNICODE_CH_LEN*/
+ var->varcharsize = var->varcharsize/UNICODE_CH_LEN;
+
+ len = var->offset * ntuples;
+ }
+ }
+ break;
case ECPGt_varchar:
len = ntuples * (var->varcharsize + sizeof(int));
break;
+ case ECPGt_uvarchar:
+ len = ntuples * (var->varcharsize*2 + sizeof(int));
+ break;
default:
len = var->offset * ntuples;
break;
@@ -416,7 +498,7 @@ ecpg_store_result(const PGresult *results, int act_field,
/* fill the variable with the tuple(s) */
if (!var->varcharsize && !var->arrsize &&
- (var->type == ECPGt_char || var->type == ECPGt_unsigned_char || var->type == ECPGt_string))
+ (var->type == ECPGt_char || var->type == ECPGt_unsigned_char || var->type == ECPGt_string || var->type == ECPGt_utext))
{
/* special mode for handling char**foo=0 */
@@ -437,7 +519,10 @@ ecpg_store_result(const PGresult *results, int act_field,
else
{
*current_string = current_data_location;
- current_data_location += len;
+ if(var->type == ECPGt_utext)
+ current_data_location += (get_utext_length(current_data_location,0)+1)*UNICODE_CH_LEN;
+ else
+ current_data_location += len;
current_string++;
}
}
@@ -772,6 +857,7 @@ ecpg_store_input(const int lineno, const bool force_indicator, const struct vari
case ECPGt_char:
case ECPGt_unsigned_char:
case ECPGt_string:
+ case ECPGt_utext:
{
/* set slen to string length if type is char * */
int slen = (var->varcharsize == 0) ? strlen((char *) var->value) : (unsigned int) var->varcharsize;
@@ -807,6 +893,7 @@ ecpg_store_input(const int lineno, const bool force_indicator, const struct vari
}
break;
case ECPGt_varchar:
+ case ECPGt_uvarchar:
{
struct ECPGgeneric_varchar *variable =
(struct ECPGgeneric_varchar *) (var->value);
@@ -1754,6 +1841,7 @@ ecpg_do_prologue(int lineno, const int compat, const int force_indicator,
enum ECPGttype type;
struct variable **list;
char *prepname;
+ int outparam = 0;
*stmt_out = NULL;
@@ -1869,7 +1957,10 @@ ecpg_do_prologue(int lineno, const int compat, const int force_indicator,
while (type != ECPGt_EORT)
{
if (type == ECPGt_EOIT)
+ {
+ outparam = 1;
list = &(stmt->outlist);
+ }
else
{
struct variable *var,
@@ -1900,6 +1991,39 @@ ecpg_do_prologue(int lineno, const int compat, const int force_indicator,
else
var->value = var->pointer;
+ if (var->type == ECPGt_utext)
+ {
+ if(outparam == 0 &&
+ ecpg_do_utext_encoding(var, PQGetEncodingFromPGconn(con->connection),lineno) == -1)
+ {
+ if ( NULL != var )
+ {
+ ecpg_free(var);
+ var = NULL;
+ }
+
+ ecpg_do_epilogue(stmt);
+
+ return false;
+ }
+ }
+ else if(var->type == ECPGt_uvarchar)
+ {
+ if(outparam == 0 &&
+ ecpg_do_uvachar_encoding(var, PQGetEncodingFromPGconn(con->connection),lineno) == -1)
+ {
+ if ( NULL != var )
+ {
+ ecpg_free(var);
+ var = NULL;
+ }
+
+ ecpg_do_epilogue(stmt);
+
+ return false;
+ }
+ }
+
/*
* negative values are used to indicate an array without given
* bounds
@@ -2049,3 +2173,217 @@ ECPGdo_descriptor(int line, const char *connection,
ECPGt_descriptor, descriptor, 0L, 0L, 0L,
ECPGt_NO_INDICATOR, NULL, 0L, 0L, 0L, ECPGt_EORT);
}
+
+/*
+ * Translate the var->value with type utext into a new value with ECPGCLIENT encoding
+ * and save it into ptr.
+ * The var->offset is changed to bytes unit.
+ * Success: return 0;
+ * Failure: return -1
+ */
+static int
+ecpg_do_utext_encoding(struct variable * var, int client_encoding,int lineno)
+{
+
+ int i;
+ int src_offset;
+ char *src_offset_start = NULL;
+ int dest_offset;
+ char *dest_offset_start = NULL;
+ char *dest;
+ int dest_length;
+ long varcharsize = 0;
+ int arrsize;
+ ConversionResult convert_ret;
+ int dest_character_len;
+
+ char *encoding = NULL;
+ char *utext_encoding = get_utext_encoding();
+
+ arrsize = var->arrsize;
+
+ /*
+ * For utext type variable, the var->offset indicate the how many bytes
+ * in one element in array.
+ * utext is 2 bytes type in windows platform
+ */
+ src_offset = var->offset;
+
+ if (utext_is_null(var->value))
+ {
+ /* Go to here means the content in the utext host varailbe is NULL,
+ * so no need convert it */
+ var->varcharsize = 1;
+ return 0;
+ }
+
+ encoding = PQGetEncodingName(client_encoding);
+ if(encoding == NULL)
+ {
+ ecpg_raise(lineno, ECPG_CLIENT_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_INTERNAL_ERROR, NULL);
+ return false;
+ }
+
+ /*
+ * For the pointer variable varcharsize is zero so have to get the real
+ * varcharsize from the get_utext_length.
+ */
+ if(var->varcharsize == 0)
+ {
+ varcharsize = get_utext_length(var->value,0);
+ if (varcharsize == 0)
+ {
+ /* Go to here means the content in the pointer is NULL,
+ * so no need convert it */
+ var->varcharsize = 1;
+ return 0;
+ }
+ else
+ src_offset = varcharsize * UNICODE_CH_LEN;
+ }
+ else
+ varcharsize = var->varcharsize;
+
+ dest_character_len = PQGenEncodingMaxLen(client_encoding);
+
+ dest_length = var->arrsize * varcharsize *dest_character_len;
+ dest_offset = varcharsize *dest_character_len;
+
+ dest = ecpg_alloc(dest_length, lineno);
+ if (!dest)
+ return -1;
+
+ memset(dest, 0, dest_length);
+
+ src_offset_start = var->value;
+ dest_offset_start = dest;
+
+ /* Convert the string offset by offset */
+ for(i=0; i<arrsize; i++)
+ {
+ convert_ret = convert_func(utext_encoding,
+ encoding,
+ src_offset_start,
+ src_offset,
+ dest_offset_start,
+ dest_offset,
+ NULL);
+
+ if(convert_ret == sourceIllegal)
+ {
+ ecpg_raise(lineno, ECPG_ENCODING_ERROR,
+ ECPG_SQLSTATE_ECPG_ENCODING_ERROR, NULL);
+
+ if ( NULL != dest )
+ {
+ ecpg_free(dest);
+ dest = NULL;
+ }
+ return -1;
+ }
+
+ src_offset_start += src_offset;
+ dest_offset_start += dest_offset;
+ }
+
+ var->offset = dest_offset;
+ var->varcharsize = dest_offset;
+ var->value = dest;
+
+ return 0;
+}
+
+
+/*
+ * Translate the var->value with type uvarchar into a new value with ECPGCLIENT encoding
+ * and save it into ptr.
+ * The var->offset is changed to bytes unit.
+ * Success: return 0;
+ * Failure: return -1
+ */
+static int
+ecpg_do_uvachar_encoding(struct variable * var, int client_encoding,int lineno)
+{
+
+ int i;
+ int src_offset;
+ char *src_offset_start = NULL;
+ int dest_offset;
+ char *dest_offset_start = NULL;
+ char *dest;
+ int dest_length;
+ int final_len = 0;
+ int arrsize;
+ ConversionResult convert_ret;
+ int dest_character_len;
+
+ char *encoding = NULL;
+ int num;
+ int num_bytes = 0;
+ char *utext_encoding = get_utext_encoding();
+
+ arrsize = var->arrsize;
+
+ dest_character_len = PQGenEncodingMaxLen(client_encoding);
+ dest_offset = (var->varcharsize *dest_character_len +sizeof(int));
+ dest_length = dest_offset *arrsize;
+
+ dest = ecpg_alloc(dest_length, lineno);
+ if (!dest)
+ return -1;
+
+ memset(dest, 0, dest_length);
+
+ src_offset = var->offset;
+ src_offset_start = var->value;
+ dest_offset_start = dest;
+
+ encoding = PQGetEncodingName(client_encoding);
+ if(encoding == NULL)
+ {
+ ecpg_raise(lineno, ECPG_CLIENT_ENCODING_ERROR, ECPG_SQLSTATE_ECPG_INTERNAL_ERROR, NULL);
+ return false;
+ }
+
+ /* Convert the string offset by offset */
+ for(i=0; i<arrsize; i++)
+ {
+ /* The num is the acctual number of character in one element */
+ num = *((int*)src_offset_start);
+
+ /* 2 means the 2 bytes occupied by every character */
+ num_bytes = num * UNICODE_CH_LEN;
+
+ convert_ret = convert_func(utext_encoding,
+ encoding,
+ src_offset_start+sizeof(int),
+ num_bytes,
+ dest_offset_start+sizeof(int),
+ dest_offset,
+ &final_len);
+
+ if(convert_ret == sourceIllegal)
+ {
+ ecpg_raise(lineno, ECPG_ENCODING_ERROR,
+ ECPG_SQLSTATE_ECPG_ENCODING_ERROR, NULL);
+
+ if ( NULL != dest )
+ {
+ ecpg_free(dest);
+ dest = NULL;
+ }
+ return -1;
+ }
+
+ *((int*)dest_offset_start)=final_len;
+ src_offset_start += src_offset;
+ dest_offset_start += dest_offset;
+ }
+
+ var->offset = dest_offset;
+ var->varcharsize = dest_offset;
+ var->value = dest;
+
+ return 0;
+}
+
diff --git a/src/interfaces/ecpg/ecpglib/extern.h b/src/interfaces/ecpg/ecpglib/extern.h
index 91c7367..a9e501f 100644
--- a/src/interfaces/ecpg/ecpglib/extern.h
+++ b/src/interfaces/ecpg/ecpglib/extern.h
@@ -122,6 +122,8 @@ struct variable
long ind_varcharsize;
long ind_arrsize;
long ind_offset;
+ int alloc_inf;
+ int ind_alloc_inf;
struct variable *next;
};
@@ -219,5 +221,6 @@ void ecpg_set_native_sqlda(int, struct sqlda_struct **, const PGresult *, int,
/* implementation-defined internal errors of ecpg */
#define ECPG_SQLSTATE_ECPG_INTERNAL_ERROR "YE000"
#define ECPG_SQLSTATE_ECPG_OUT_OF_MEMORY "YE001"
+#define ECPG_SQLSTATE_ECPG_ENCODING_ERROR "YE002"
#endif /* _ECPG_LIB_EXTERN_H */
diff --git a/src/interfaces/ecpg/ecpglib/misc.c b/src/interfaces/ecpg/ecpglib/misc.c
index 2084d7f..dc4331e 100644
--- a/src/interfaces/ecpg/ecpglib/misc.c
+++ b/src/interfaces/ecpg/ecpglib/misc.c
@@ -369,6 +369,21 @@ ECPGset_noind_null(enum ECPGttype type, void *ptr)
case ECPGt_timestamp:
memset((char *) ptr, 0xff, sizeof(timestamp));
break;
+ case ECPGt_utext:
+#if defined(WIN32)
+ memset(ptr, 0x00, 2);
+#else
+ memset(ptr, 0x00, 4);
+#endif
+ break;
+ case ECPGt_uvarchar:
+#if defined(WIN32)
+ memset(((struct ECPGgeneric_varchar *) ptr)->arr, 0x00, 2);
+#else
+ memset(((struct ECPGgeneric_varchar *) ptr)->arr, 0x00, 4);
+#endif
+ ((struct ECPGgeneric_varchar *) ptr)->len = 0;
+ break;
default:
break;
}
@@ -442,6 +457,18 @@ ECPGis_noind_null(enum ECPGttype type, void *ptr)
case ECPGt_timestamp:
return _check(ptr, sizeof(timestamp));
break;
+ case ECPGt_utext:
+#if defined(WIN32)
+ if (*((short int *) ptr) == 0)
+#else
+ if (*((int *) ptr) == 0)
+#endif
+ return true;
+ break;
+ case ECPGt_uvarchar:
+ if (((struct ECPGgeneric_varchar *) ptr)->len == 0)
+ return true;
+ break;
default:
break;
}
diff --git a/src/interfaces/ecpg/ecpglib/typename.c b/src/interfaces/ecpg/ecpglib/typename.c
index 48587e4..aec3ac2 100644
--- a/src/interfaces/ecpg/ecpglib/typename.c
+++ b/src/interfaces/ecpg/ecpglib/typename.c
@@ -47,6 +47,10 @@ ecpg_type_name(enum ECPGttype typ)
return "bool";
case ECPGt_varchar:
return "varchar";
+ case ECPGt_utext:
+ return "utext";
+ case ECPGt_uvarchar:
+ return "uvarchar";
case ECPGt_char_variable:
return "char";
case ECPGt_decimal:
diff --git a/src/interfaces/ecpg/include/convertutf.h b/src/interfaces/ecpg/include/convertutf.h
new file mode 100644
index 0000000..7bddf40
--- /dev/null
+++ b/src/interfaces/ecpg/include/convertutf.h
@@ -0,0 +1,42 @@
+#ifndef ConvertUTF_INCLUDED
+#define ConvertUTF_INCLUDED
+#include <stddef.h>
+#include "c.h"
+
+/* define unicode character length by bytes */
+#if defined(WIN32)
+#define UNICODE_CH_LEN 2
+#else
+#define UNICODE_CH_LEN 4
+#endif
+
+typedef enum
+{
+ conversionOK, /* conversion successful */
+ sourceExhausted, /* partial character in source, but hit end */
+ targetExhausted, /* insuff. room in target for conversion */
+ sourceIllegal, /* source sequence is illegal/malformed */
+ conversionUnsupported /* Don't support this type convert */
+} ConversionResult;
+
+/* This is for C++ and does no harm in C */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bool utext_is_null(char *source);
+long get_utext_length(char *source, unsigned int maxlen);
+
+ConversionResult convert_func(char *from_code, char *to_code,
+ char *from, size_t from_len,
+ char *to, size_t to_len, int *real_len);
+
+int get_converted_length(char *from_code, char *to_code, char *from, size_t from_len);
+char *get_utext_encoding(void);
+#ifdef __cplusplus
+}
+#endif
+
+/* --------------------------------------------------------------------- */
+
+#endif /* ConvertUTF_INCLUDED */
diff --git a/src/interfaces/ecpg/include/ecpgerrno.h b/src/interfaces/ecpg/include/ecpgerrno.h
index c4bc526..6d13065 100644
--- a/src/interfaces/ecpg/include/ecpgerrno.h
+++ b/src/interfaces/ecpg/include/ecpgerrno.h
@@ -44,6 +44,8 @@
#define ECPG_UNKNOWN_DESCRIPTOR_ITEM -242
#define ECPG_VAR_NOT_NUMERIC -243
#define ECPG_VAR_NOT_CHAR -244
+#define ECPG_ENCODING_ERROR -250
+#define ECPG_CLIENT_ENCODING_ERROR -301
/* finally the backend error messages, they start at 400 */
#define ECPG_PGSQL -400
diff --git a/src/interfaces/ecpg/include/ecpglib.h b/src/interfaces/ecpg/include/ecpglib.h
index 536b750..d546c49 100644
--- a/src/interfaces/ecpg/include/ecpglib.h
+++ b/src/interfaces/ecpg/include/ecpglib.h
@@ -11,13 +11,24 @@
#include "ecpgtype.h"
#include "sqlca.h"
#include <string.h>
-
+#if defined(WIN32)
+#include <windows.h>
+#endif
#ifdef ENABLE_NLS
extern char *ecpg_gettext(const char *msgid) pg_attribute_format_arg(1);
#else
#define ecpg_gettext(x) (x)
#endif
+#if defined ECPG_ENABLE_UTEXT
+#if defined(WIN32)
+typedef WCHAR utext;
+#else
+#include <wchar.h>
+typedef wchar_t utext;
+#endif
+#endif
+
#ifndef __cplusplus
#ifndef bool
#define bool char
diff --git a/src/interfaces/ecpg/include/ecpgtype.h b/src/interfaces/ecpg/include/ecpgtype.h
index 38fb3b6..0316a84 100644
--- a/src/interfaces/ecpg/include/ecpgtype.h
+++ b/src/interfaces/ecpg/include/ecpgtype.h
@@ -63,7 +63,9 @@ enum ECPGttype
ECPGt_EORT, /* End of result types. */
ECPGt_NO_INDICATOR, /* no indicator */
ECPGt_string, /* trimmed (char *) type */
- ECPGt_sqlda /* C struct descriptor */
+ ECPGt_sqlda, /* C struct descriptor */
+ ECPGt_utext,
+ ECPGt_uvarchar
};
/* descriptor items */
@@ -88,7 +90,7 @@ enum ECPGdtype
ECPGd_cardinality
};
-#define IS_SIMPLE_TYPE(type) (((type) >= ECPGt_char && (type) <= ECPGt_interval) || ((type) == ECPGt_string))
+#define IS_SIMPLE_TYPE(type) (((type) >= ECPGt_char && (type) <= ECPGt_interval) || ((type) == ECPGt_string) || ((type) == ECPGt_utext) || ((type) == ECPGt_uvarchar))
/* we also have to handle different statement types */
enum ECPG_statement_type
diff --git a/src/interfaces/ecpg/preproc/ecpg.c b/src/interfaces/ecpg/preproc/ecpg.c
index 536185f..0efdaff 100644
--- a/src/interfaces/ecpg/preproc/ecpg.c
+++ b/src/interfaces/ecpg/preproc/ecpg.c
@@ -10,6 +10,7 @@
#include "getopt_long.h"
#include "extern.h"
+#include "mb/pg_wchar.h"
int ret_value = 0;
bool autocommit = false,
@@ -18,7 +19,8 @@ bool autocommit = false,
force_indicator = true,
questionmarks = false,
regression_mode = false,
- auto_prepare = false;
+ auto_prepare = false,
+ enable_utext = false;
char *output_filename;
@@ -54,6 +56,7 @@ help(const char *progname)
" \"no_indicator\", \"prepare\", \"questionmarks\"\n"));
printf(_(" --regression run in regression testing mode\n"));
printf(_(" -t turn on autocommit of transactions\n"));
+ printf(_(" --enable-utext enable unicode feature\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nIf no output file is specified, the name is formed by adding .c to the\n"
@@ -112,11 +115,13 @@ add_preprocessor_define(char *define)
}
#define ECPG_GETOPT_LONG_REGRESSION 1
+#define ECPG_GETOPT_LONG_ENABLE_UTEXT 2
int
main(int argc, char *const argv[])
{
static struct option ecpg_options[] = {
{"regression", no_argument, NULL, ECPG_GETOPT_LONG_REGRESSION},
+ {"enable-utext", no_argument, NULL, ECPG_GETOPT_LONG_ENABLE_UTEXT},
{NULL, 0, NULL, 0}
};
@@ -237,6 +242,9 @@ main(int argc, char *const argv[])
fprintf(stderr, _("%s: parser debug support (-d) not available\n"),
progname);
#endif
+ case ECPG_GETOPT_LONG_ENABLE_UTEXT:
+ enable_utext = true;
+ break;
break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), argv[0]);
@@ -434,7 +442,11 @@ main(int argc, char *const argv[])
if (header_mode == false)
{
- fprintf(base_yyout, "/* These include files are added by the preprocessor */\n#include <ecpglib.h>\n#include <ecpgerrno.h>\n#include <sqlca.h>\n");
+ fprintf(base_yyout, "/* These include files are added by the preprocessor */\n");
+
+ if(enable_utext)
+ fprintf(base_yyout, "#define ECPG_ENABLE_UTEXT 1\n");
+ fprintf(base_yyout, "#include <ecpglib.h>\n#include <ecpgerrno.h>\n#include <sqlca.h>\n");
/* add some compatibility headers */
if (INFORMIX_MODE)
diff --git a/src/interfaces/ecpg/preproc/ecpg.header b/src/interfaces/ecpg/preproc/ecpg.header
index 8921bcb..304aea4 100644
--- a/src/interfaces/ecpg/preproc/ecpg.header
+++ b/src/interfaces/ecpg/preproc/ecpg.header
@@ -6,6 +6,7 @@
#include "extern.h"
#include "ecpg_config.h"
+#include "mb/pg_wchar.h"
#include <unistd.h>
/* Location tracking support --- simpler than bison's default */
@@ -46,6 +47,7 @@ static char pacounter_buffer[sizeof(int) * CHAR_BIT * 10 / 3]; /* a rough guess
static struct this_type actual_type[STRUCT_DEPTH];
static char *actual_startline[STRUCT_DEPTH];
static int varchar_counter = 1;
+static int uvarchar_counter = 1;
/* temporarily store struct members while creating the data structure */
struct ECPGstruct_member *struct_member_list[STRUCT_DEPTH] = { NULL };
@@ -137,7 +139,7 @@ cat2_str(char *str1, char *str2)
char * res_str = (char *)mm_alloc(strlen(str1) + strlen(str2) + 2);
strcpy(res_str, str1);
- if (strlen(str1) != 0 && strlen(str2) != 0)
+ if (strlen(str1) != 0 && strlen(str2) != 0 && strcmp(str1,"L") != 0)
strcat(res_str, " ");
strcat(res_str, str2);
free(str1);
@@ -288,7 +290,9 @@ adjust_outofscope_cursor_vars(struct cursor *cur)
else if ((ptr->variable->type->type != ECPGt_varchar
&& ptr->variable->type->type != ECPGt_char
&& ptr->variable->type->type != ECPGt_unsigned_char
- && ptr->variable->type->type != ECPGt_string)
+ && ptr->variable->type->type != ECPGt_string
+ && ptr->variable->type->type != ECPGt_utext
+ && ptr->variable->type->type != ECPGt_uvarchar)
&& atoi(ptr->variable->type->size) > 1)
{
newvar = new_variable(cat_str(4, mm_strdup("("),
@@ -304,7 +308,9 @@ adjust_outofscope_cursor_vars(struct cursor *cur)
else if ((ptr->variable->type->type == ECPGt_varchar
|| ptr->variable->type->type == ECPGt_char
|| ptr->variable->type->type == ECPGt_unsigned_char
- || ptr->variable->type->type == ECPGt_string)
+ || ptr->variable->type->type == ECPGt_string
+ || ptr->variable->type->type == ECPGt_utext
+ || ptr->variable->type->type == ECPGt_uvarchar)
&& atoi(ptr->variable->type->size) > 1)
{
newvar = new_variable(cat_str(4, mm_strdup("("),
@@ -315,7 +321,7 @@ adjust_outofscope_cursor_vars(struct cursor *cur)
ptr->variable->type->size,
ptr->variable->type->counter),
0);
- if (ptr->variable->type->type == ECPGt_varchar)
+ if (ptr->variable->type->type == ECPGt_varchar || ptr->variable->type->type == ECPGt_uvarchar)
var_ptr = true;
}
else if (ptr->variable->type->type == ECPGt_struct
@@ -565,6 +571,8 @@ add_typedef(char *name, char *dimension, char *length, enum ECPGttype type_enum,
type_enum != ECPGt_char &&
type_enum != ECPGt_unsigned_char &&
type_enum != ECPGt_string &&
+ type_enum != ECPGt_utext &&
+ type_enum != ECPGt_uvarchar &&
atoi(this->type->type_index) >= 0)
mmerror(PARSE_ERROR, ET_ERROR, "multidimensional arrays for simple data types are not supported");
diff --git a/src/interfaces/ecpg/preproc/ecpg.tokens b/src/interfaces/ecpg/preproc/ecpg.tokens
index 68ba925..2072382 100644
--- a/src/interfaces/ecpg/preproc/ecpg.tokens
+++ b/src/interfaces/ecpg/preproc/ecpg.tokens
@@ -21,7 +21,8 @@
S_DOTPOINT S_EQUAL S_EXTERN S_INC S_LSHIFT S_MEMPOINT
S_MEMBER S_MOD S_MUL S_NEQUAL S_OR S_REGISTER S_RSHIFT
S_STATIC S_SUB S_VOLATILE
- S_TYPEDEF
+ S_TYPEDEF UTEXT
%token CSTRING CVARIABLE CPP_LINE IP
%token DOLCONST ECONST NCONST UCONST UIDENT
+%token UVARCHAR
diff --git a/src/interfaces/ecpg/preproc/ecpg.trailer b/src/interfaces/ecpg/preproc/ecpg.trailer
index f60a620..4984aec 100644
--- a/src/interfaces/ecpg/preproc/ecpg.trailer
+++ b/src/interfaces/ecpg/preproc/ecpg.trailer
@@ -186,7 +186,7 @@ user_name: RoleId
type = argsinsert->variable->type->u.element->type;
/* handle varchars */
- if (type == ECPGt_varchar)
+ if (type == ECPGt_varchar || type == ECPGt_uvarchar)
$$ = make2_str(mm_strdup(argsinsert->variable->name), mm_strdup(".arr"));
else
$$ = mm_strdup(argsinsert->variable->name);
@@ -213,9 +213,11 @@ char_variable: cvariable
case ECPGt_char:
case ECPGt_unsigned_char:
case ECPGt_string:
+ case ECPGt_utext:
$$ = $1;
break;
case ECPGt_varchar:
+ case ECPGt_uvarchar:
$$ = make2_str($1, mm_strdup(".arr"));
break;
default:
@@ -550,6 +552,22 @@ var_type: simple_type
$$.type_index = mm_strdup("-1");
$$.type_sizeof = NULL;
}
+ else if (strcmp($1, "uvarchar") == 0 && enable_utext)
+ {
+ $$.type_enum = ECPGt_uvarchar;
+ $$.type_str = EMPTY;
+ $$.type_dimension = mm_strdup("-1");
+ $$.type_index = mm_strdup("-1");
+ $$.type_sizeof = NULL;
+ }
+ else if (strcmp($1, "utext") == 0 && enable_utext)
+ {
+ $$.type_enum = ECPGt_utext;
+ $$.type_str = EMPTY;
+ $$.type_dimension = mm_strdup("-1");
+ $$.type_index = mm_strdup("-1");
+ $$.type_sizeof = NULL;
+ }
else if (strcmp($1, "float") == 0)
{
$$.type_enum = ECPGt_float;
@@ -627,7 +645,7 @@ var_type: simple_type
/* this is for typedef'ed types */
struct typedefs *this = get_typedef($1);
- $$.type_str = (this->type->type_enum == ECPGt_varchar) ? EMPTY : mm_strdup(this->name);
+ $$.type_str = (this->type->type_enum == ECPGt_varchar || this->type->type_enum == ECPGt_uvarchar) ? EMPTY : mm_strdup(this->name);
$$.type_enum = this->type->type_enum;
$$.type_dimension = this->type->type_dimension;
$$.type_index = this->type->type_index;
@@ -838,7 +856,7 @@ variable_list: variable
{ $$ = $1; }
| variable_list ',' variable
{
- if (actual_type[struct_level].type_enum == ECPGt_varchar)
+ if (actual_type[struct_level].type_enum == ECPGt_varchar || actual_type[struct_level].type_enum == ECPGt_uvarchar)
$$ = cat_str(3, $1, mm_strdup(";"), $3);
else
$$ = cat_str(3, $1, mm_strdup(","), $3);
@@ -890,7 +908,52 @@ variable: opt_pointer ECPGColLabel opt_array_bounds opt_bit_field opt_initialize
$$ = cat_str(8, make2_str(mm_strdup(" struct varchar_"), vcn), mm_strdup(" { int len; char arr["), mm_strdup(length), mm_strdup("]; } "), mm_strdup($2), dim_str, $4, $5);
varchar_counter++;
break;
+ case ECPGt_uvarchar:
+ if (atoi(dimension) < 0)
+ type = ECPGmake_simple_type(actual_type[struct_level].type_enum, length, uvarchar_counter);
+ else
+ type = ECPGmake_array_type(ECPGmake_simple_type(actual_type[struct_level].type_enum, length, uvarchar_counter), dimension);
+ if (strcmp(dimension, "0") == 0 || abs(atoi(dimension)) == 1)
+ dim_str=mm_strdup("");
+ else
+ dim_str=cat_str(3, mm_strdup("["), mm_strdup(dimension), mm_strdup("]"));
+ /* cannot check for atoi <= 0 because a defined constant will yield 0 here as well */
+ if (atoi(length) < 0 || strcmp(length, "0") == 0)
+ mmerror(PARSE_ERROR, ET_ERROR, "pointers to uvarchar are not implemented");
+
+ /* make sure varchar struct name is unique by adding a unique counter to its definition */
+ vcn = (char *) mm_alloc(sizeof(int) * CHAR_BIT * 10 / 3);
+ sprintf(vcn, "%d", uvarchar_counter);
+ if (strcmp(dimension, "0") == 0)
+ $$ = cat_str(7, make2_str(mm_strdup(" struct uvarchar_"), vcn), mm_strdup(" { int len; utext arr["), mm_strdup(length), mm_strdup("]; } *"), mm_strdup($2), $4, $5);
+ else
+ $$ = cat_str(8, make2_str(mm_strdup(" struct uvarchar_"), vcn), mm_strdup(" { int len; utext arr["), mm_strdup(length), mm_strdup("]; } "), mm_strdup($2), dim_str, $4, $5);
+ uvarchar_counter++;
+ break;
+ case ECPGt_utext:
+ if (atoi(dimension) == -1)
+ {
+ int i = strlen($5);
+
+ if (atoi(length) == -1 && i > 0) /* char <var>[] = "string" */
+ {
+ /* if we have an initializer but no string size set, let's use the initializer's length */
+ free(length);
+ length = mm_alloc(i+sizeof("sizeof()"));
+ #if defined(WIN32)
+ sprintf(length, "sizeof(%s)/2", $5+2);
+ #else
+ sprintf(length, "sizeof(%s)/4", $5+2);
+ #endif
+ }
+ type = ECPGmake_simple_type(actual_type[struct_level].type_enum, length, 0);
+ }
+ else
+ type = ECPGmake_array_type(ECPGmake_simple_type(actual_type[struct_level].type_enum, length, 0), dimension);
+
+ $$ = cat_str(6, mm_strdup("utext"), $1, mm_strdup($2), $3.str, $4, $5);
+ break;
case ECPGt_char:
case ECPGt_unsigned_char:
case ECPGt_string:
@@ -1354,6 +1417,7 @@ ECPGVar: SQL_VAR
break;
case ECPGt_varchar:
+ case ECPGt_uvarchar:
if (atoi(dimension) == -1)
type = ECPGmake_simple_type($5.type_enum, length, 0);
else
@@ -1363,6 +1427,7 @@ ECPGVar: SQL_VAR
case ECPGt_char:
case ECPGt_unsigned_char:
case ECPGt_string:
+ case ECPGt_utext:
if (atoi(dimension) == -1)
type = ECPGmake_simple_type($5.type_enum, length, 0);
else
@@ -1504,6 +1569,8 @@ ECPGKeywords_vanames: SQL_BREAK { $$ = mm_strdup("break"); }
| SQL_SQLPRINT { $$ = mm_strdup("sqlprint"); }
| SQL_SQLWARNING { $$ = mm_strdup("sqlwarning"); }
| SQL_STOP { $$ = mm_strdup("stop"); }
+ | UVARCHAR { $$ = mm_strdup("uvarchar"); }
+ | UTEXT { $$ = mm_strdup("utext"); }
;
ECPGKeywords_rest: SQL_CONNECT { $$ = mm_strdup("connect"); }
@@ -1854,6 +1921,8 @@ c_anything: ecpg_ident { $$ = $1; }
| TO { $$ = mm_strdup("to"); }
| UNION { $$ = mm_strdup("union"); }
| VARCHAR { $$ = mm_strdup("varchar"); }
+ | UVARCHAR { $$ = mm_strdup("uvarchar"); }
+ | UTEXT { $$ = mm_strdup("utext"); }
| '[' { $$ = mm_strdup("["); }
| ']' { $$ = mm_strdup("]"); }
| '=' { $$ = mm_strdup("="); }
diff --git a/src/interfaces/ecpg/preproc/ecpg.type b/src/interfaces/ecpg/preproc/ecpg.type
index 9497b91..3a95f9f 100644
--- a/src/interfaces/ecpg/preproc/ecpg.type
+++ b/src/interfaces/ecpg/preproc/ecpg.type
@@ -128,6 +128,8 @@
%type <str> SCONST
%type <str> UCONST
%type <str> UIDENT
+%type <str> UTEXT
+%type <str> UVARCHAR
%type <struct_union> s_struct_union_symbol
diff --git a/src/interfaces/ecpg/preproc/extern.h b/src/interfaces/ecpg/preproc/extern.h
index 2c35426..d612977 100644
--- a/src/interfaces/ecpg/preproc/extern.h
+++ b/src/interfaces/ecpg/preproc/extern.h
@@ -25,7 +25,8 @@ extern bool autocommit,
force_indicator,
questionmarks,
regression_mode,
- auto_prepare;
+ auto_prepare,
+ enable_utext;
extern int braces_open,
ret_value,
struct_level,
diff --git a/src/interfaces/ecpg/preproc/pgc.l b/src/interfaces/ecpg/preproc/pgc.l
index fc450f3..1acea33 100644
--- a/src/interfaces/ecpg/preproc/pgc.l
+++ b/src/interfaces/ecpg/preproc/pgc.l
@@ -246,6 +246,8 @@ less_equals "<="
greater_equals ">="
less_greater "<>"
not_equals "!="
+utext [uU][tT][eE][xX][tT]
+uvarchar [uU][vV][aA][rR][cC][hH][aA][rR]
/*
* "self" is the set of chars that should be returned as single-character
@@ -770,6 +772,10 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+
base_yylval.str = mm_strdup(yytext+1);
return CVARIABLE;
}
+<C>{utext} { return(UTEXT); }
+<C,C_SQL>{uvarchar} {
+ return(UVARCHAR);
+ }
<SQL>{identifier} {
const ScanKeyword *keyword;
diff --git a/src/interfaces/ecpg/preproc/type.c b/src/interfaces/ecpg/preproc/type.c
index 256a3c3..649fcb2 100644
--- a/src/interfaces/ecpg/preproc/type.c
+++ b/src/interfaces/ecpg/preproc/type.c
@@ -175,6 +175,12 @@ get_type(enum ECPGttype type)
break;
case ECPGt_varchar:
return "ECPGt_varchar";
+ case ECPGt_utext:
+ /* FEP Feature - Unicode */
+ return ("ECPGt_utext");
+ case ECPGt_uvarchar:
+ /* FEP Feature - Unicode */
+ return ("ECPGt_uvarchar");
case ECPGt_NO_INDICATOR: /* no indicator */
return "ECPGt_NO_INDICATOR";
break;
@@ -454,6 +460,50 @@ ECPGdump_a_simple(FILE *o, const char *name, enum ECPGttype type,
else
sprintf(offset, "sizeof(struct varchar)");
break;
+ case ECPGt_uvarchar:
+
+ /*
+ * we have to use the pointer except for arrays with given
+ * bounds
+ */
+ if (((atoi(arrsize) > 0) ||
+ (atoi(arrsize) == 0 && strcmp(arrsize, "0") != 0)) &&
+ size == NULL)
+ sprintf(variable, "(%s%s)", prefix ? prefix : "", name);
+ else
+ sprintf(variable, "&(%s%s)", prefix ? prefix : "", name);
+
+ /*
+ * If we created a varchar structure automatically, counter is
+ * greater than 0.
+ */
+ if (counter)
+ sprintf(offset, "sizeof(struct uvarchar_%d)", counter);
+ else
+ sprintf(offset, "sizeof(struct uvarchar)");
+ break;
+ case ECPGt_utext:
+ {
+ char *sizeof_name = "utext";
+
+ /*
+ * we have to use the pointer except for arrays with given
+ * bounds, ecpglib will distinguish between * and []
+ */
+ if ((atoi(varcharsize) > 1 ||
+ (atoi(arrsize) > 0) ||
+ (atoi(varcharsize) == 0 && strcmp(varcharsize, "0") != 0) ||
+ (atoi(arrsize) == 0 && strcmp(arrsize, "0") != 0))
+ && size == NULL)
+ {
+ sprintf(variable, "(%s%s)", prefix ? prefix : "", name);
+ }
+ else
+ sprintf(variable, "&(%s%s)", prefix ? prefix : "", name);
+
+ sprintf(offset, "(%s)*sizeof(%s)", strcmp(varcharsize, "0") == 0 ? "1" : varcharsize, sizeof_name);
+ break;
+ }
case ECPGt_char:
case ECPGt_unsigned_char:
case ECPGt_char_variable:
diff --git a/src/interfaces/ecpg/preproc/variable.c b/src/interfaces/ecpg/preproc/variable.c
index 39bf3b2..8f681c9 100644
--- a/src/interfaces/ecpg/preproc/variable.c
+++ b/src/interfaces/ecpg/preproc/variable.c
@@ -535,7 +535,7 @@ adjust_array(enum ECPGttype type_enum, char **dimension, char **length, char *ty
"multilevel pointers (more than 2 levels) are not supported; found %d levels", pointer_len),
pointer_len);
- if (pointer_len > 1 && type_enum != ECPGt_char && type_enum != ECPGt_unsigned_char && type_enum != ECPGt_string)
+ if (pointer_len > 1 && type_enum != ECPGt_char && type_enum != ECPGt_unsigned_char && type_enum != ECPGt_string && type_enum != ECPGt_utext)
mmfatal(PARSE_ERROR, "pointer to pointer is not supported for this data type");
if (pointer_len > 1 && (atoi(*length) >= 0 || atoi(*dimension) >= 0))
@@ -560,6 +560,7 @@ adjust_array(enum ECPGttype type_enum, char **dimension, char **length, char *ty
break;
case ECPGt_varchar:
+ case ECPGt_uvarchar:
/* pointer has to get dimension 0 */
if (pointer_len)
*dimension = mm_strdup("0");
@@ -573,6 +574,7 @@ adjust_array(enum ECPGttype type_enum, char **dimension, char **length, char *ty
break;
case ECPGt_char:
+ case ECPGt_utext:
case ECPGt_unsigned_char:
case ECPGt_string:
/* char ** */
diff --git a/src/interfaces/ecpg/test/ecpg_schedule b/src/interfaces/ecpg/test/ecpg_schedule
index cff4eeb..04d80a5 100644
--- a/src/interfaces/ecpg/test/ecpg_schedule
+++ b/src/interfaces/ecpg/test/ecpg_schedule
@@ -49,6 +49,8 @@ test: sql/quote
test: sql/show
test: sql/insupd
test: sql/parser
+test: sql/utext
+test: sql/uvarchar
test: thread/thread
test: thread/thread_implicit
test: thread/prep
diff --git a/src/interfaces/ecpg/test/ecpg_schedule_tcp b/src/interfaces/ecpg/test/ecpg_schedule_tcp
index 77481b5..fdd0e14 100644
--- a/src/interfaces/ecpg/test/ecpg_schedule_tcp
+++ b/src/interfaces/ecpg/test/ecpg_schedule_tcp
@@ -47,6 +47,8 @@ test: sql/quote
test: sql/show
test: sql/insupd
test: sql/parser
+test: sql/utext
+test: sql/uvarchar
test: thread/thread
test: thread/thread_implicit
test: thread/prep
diff --git a/src/interfaces/ecpg/test/expected/sql-utext.c b/src/interfaces/ecpg/test/expected/sql-utext.c
new file mode 100644
index 0000000..fdaec97
--- /dev/null
+++ b/src/interfaces/ecpg/test/expected/sql-utext.c
@@ -0,0 +1,4451 @@
+/* Processed by ecpg (regression mode) */
+/* These include files are added by the preprocessor */
+#define ECPG_ENABLE_UTEXT 1
+#include <ecpglib.h>
+#include <ecpgerrno.h>
+#include <sqlca.h>
+/* End of automatic include section */
+#define ECPGdebug(X,Y) ECPGdebug((X)+100,(Y))
+
+#line 1 "utext.pgc"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <wchar.h>
+
+#line 1 "regression.h"
+
+
+
+
+
+
+#line 5 "utext.pgc"
+
+
+#define test(msg) printf("\n%s\n",msg)
+#define VAR_SIZE 20
+#define ARRAY_SIZE 4
+#define P_VAR_SIZE 22
+
+
+/* Following is UTF8 and UTF16/UTF32 characters mapping table */
+/*太𠮷𠜱平洋𠱓大西洋印度洋北冰洋
+utf16
+0x592A 0xD842 0xDFB7 0xD841 0xDF31 0x5E73 0x6D0B 0xD843
+0xDC53 0x5927 0x897F 0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317
+0x51B0 0x6D0B 0x0000,0x0000
+
+utf32
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B
+*/
+
+/*足球篮球羽毛球乒乓球橄榄球棒球冰球
+utf16
+0x8DB3,0x7403,0x7BEE,0x7403,0x7FBD,0x6BDB,0x7403,0x4E52,
+0x4E53,0x7403,0x6A44,0x6984,0x7403,0x68D2,0x7403,0x51B0,
+0x7403,0x0000,0x0000,0x0000
+
+utf32
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403
+*/
+
+/*世界杯每隔四年就会举行一次每次𠲖个球队
+utf16
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0xD843
+0xDC96 0x4E2A 0x7403 0x961F
+
+utf32
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F
+*/
+
+/* 亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲
+0x4E9A,0x6D32,0x6B27,0x6D32,0x975E,0x6D32,0x5927,0x6D0B,
+0x6D32,0x5317,0x7F8E,0x6D32,0x5357,0x7F8E,0x6D32,0x5357,
+0x6781,0x6D32,0x6CA1,0x6709,0x5317,0x6781,0x6D32
+*/
+/* 太𠮷
+0x592A 0xD842 0xDFB7
+*/
+void init_var(void);
+void test_var_1(void);
+void test_var_2(void);
+void test_var_3(void);
+void test_var_4(void);
+void test_var_5(void);
+void test_var_6(void);
+void test_var_7(void);
+void test_var_8(void);
+void test_var_9(void);
+void test_var_10(void);
+void test_var_11(void);
+void test_var_12(void);
+void test_var_13(void);
+void test_var_14(void);
+void test_var_15(void);
+void test_var_16(void);
+void test_array_1(void);
+void test_array_2(void);
+void test_array_3(void);
+void test_array_4(void);
+void test_array_5(void);
+void test_array_6(void);
+void test_array_7(void);
+void test_array_8(void);
+void test_array_9(void);
+void test_array_10(void);
+void test_array_11(void);
+void test_array_12(void);
+void test_array_13(void);
+void test_array_14(void);
+void test_array_15(void);
+void test_array_16(void);
+void test_pvar_1(void);
+void test_pvar_2(void);
+void test_pvar_3(void);
+void test_pvar_4(void);
+void test_pvar_5(void);
+void test_pvar_6(void);
+void test_pvar_7(void);
+void test_pvar_8(void);
+void test_pvar_9(void);
+void test_pvar_10(void);
+void test_pvar_11(void);
+void test_pvar_12(void);
+void test_pvar_13(void);
+void test_pvar_14(void);
+void test_pvar_15(void);
+void test_pvar_16(void);
+void test_pvar_17(void);
+void test_pvar_18(void);
+void test_pvar_19(void);
+void test_pp_var_1(void);
+void test_pp_var_2(void);
+void test_pp_var_3(void);
+void test_desc_1(void);
+void test_all(void);
+
+void print_utext(utext *utext_var);
+void print_utext_str(utext *utext_var);
+void print_utext_ind(int utext_var_ind);
+void print_utext_size(utext *utext_var,int size);
+void print_array(void *array, int array_size, int var_size);
+void print_array_with_index(int index);
+int test_init(void);
+void test_finish(void);
+void init_table_value(void);
+
+/* exec sql begin declare section */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+#line 126 "utext.pgc"
+ int utext_var_size = 20 ;
+
+#line 127 "utext.pgc"
+ int utext_array_size = 4 ;
+
+#line 128 "utext.pgc"
+ int count = 0 ;
+
+#line 129 "utext.pgc"
+ int total_tuples = 0 ;
+
+#line 130 "utext.pgc"
+ int i ;
+
+#line 131 "utext.pgc"
+ char char_var [ 10 ] = { 0xF0 , 0x90 , 0x90 , 0xB7 } ;
+
+#line 133 "utext.pgc"
+ utext utext_var [ VAR_SIZE ] ;
+
+#line 134 "utext.pgc"
+ utext utext_array [ ARRAY_SIZE ] [ VAR_SIZE ] ;
+
+#line 135 "utext.pgc"
+ utext utext_input_var [ 20 ] = { 0x592a , 0xd842 , 0xdfb7 , 0x0000 } ;
+
+#line 136 "utext.pgc"
+ utext * p_utext_var = NULL ;
+
+#line 139 "utext.pgc"
+ int utext_var_ind ;
+
+#line 140 "utext.pgc"
+ int count_array [ 4 ] = { 1 , 2 , 3 , 4 } ;
+/* exec sql end declare section */
+#line 142 "utext.pgc"
+
+
+void print_utext(utext *utext_var)
+{
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<VAR_SIZE; i++)
+ {
+ printf ("0x%08X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void print_utext_str(utext *utext_var)
+{
+ int i=0;
+ printf("======print utext_var_str content======\n");
+ if (utext_var==0)
+ {
+ printf("utext_var is NULL\n");
+ printf("\n======End utext_var_str content======\n");
+ return;
+ }
+ while(utext_var[i] != 0)
+ {
+ printf ("0x%08X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ i++;
+ if(i>100)
+ break;
+ }
+ printf("\n======End utext_var_str content======\n");
+}
+
+void print_utext_size(utext *utext_var,int size)
+{
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<size; i++)
+ {
+ printf ("0x%08X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void print_utext_ind(int utext_var_ind)
+{
+ printf("utext_var_ind = %d\n",utext_var_ind);
+}
+void print_array(void *array, int array_size, int var_size)
+{
+ int i,j;
+
+ for(i=0; i<array_size; i++)
+ {
+ utext *utext_array = &((utext*)array)[i*var_size];
+
+ printf ("---->array[%d]:\n", i);
+
+ for(j=0; j<var_size; j++)
+ {
+ printf ("0x%08X ", utext_array[j]);
+ if(j>6 && (j+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+ }
+
+ printf("\n");
+}
+
+int test_init()
+{
+ p_utext_var = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ if(!p_utext_var)
+ {
+ printf("Error: failed allco memory for p_utext_var. \n");
+ return -1;
+ }
+
+ { ECPGconnect(__LINE__, 0, "ecpg1_regression" , NULL, NULL , NULL, 0); }
+#line 227 "utext.pgc"
+
+
+ { ECPGsetcommit(__LINE__, "on", NULL);}
+#line 229 "utext.pgc"
+
+ /* exec sql whenever sql_warning sqlprint ; */
+#line 230 "utext.pgc"
+
+ /* exec sql whenever sqlerror sqlprint ; */
+#line 231 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "set client_encoding = 'UTF8'", ECPGt_EOIT, ECPGt_EORT);
+#line 233 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 233 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 233 "utext.pgc"
+
+
+ //initialization of test table
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "create table if not exists tb1 ( Item varchar , count integer )", ECPGt_EOIT, ECPGt_EORT);
+#line 236 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 236 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 236 "utext.pgc"
+
+
+ init_table_value();
+
+ return 0;
+}
+
+void test_finish()
+{
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "drop table tb1", ECPGt_EOIT, ECPGt_EORT);
+#line 245 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 245 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 245 "utext.pgc"
+
+ { ECPGdisconnect(__LINE__, "ALL");
+#line 246 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 246 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 246 "utext.pgc"
+
+}
+
+void init_table_value()
+{
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "truncate tb1", ECPGt_EOIT, ECPGt_EORT);
+#line 251 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 251 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 251 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋' , 1 )", ECPGt_EOIT, ECPGt_EORT);
+#line 253 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 253 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 253 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( '足球篮球羽毛球乒乓球橄榄球棒球冰球' , 2 )", ECPGt_EOIT, ECPGt_EORT);
+#line 254 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 254 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 254 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( '世界杯每隔四年就会举行一次每次𠲖个球队' , 3 )", ECPGt_EOIT, ECPGt_EORT);
+#line 255 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 255 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 255 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( '亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲' , 4 )", ECPGt_EOIT, ECPGt_EORT);
+#line 256 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 256 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 256 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 ( count ) values ( 8 )", ECPGt_EOIT, ECPGt_EORT);
+#line 257 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 257 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 257 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 ( count ) values ( 9 )", ECPGt_EOIT, ECPGt_EORT);
+#line 258 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 258 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 258 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 ( count ) values ( 10 )", ECPGt_EOIT, ECPGt_EORT);
+#line 259 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 259 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 259 "utext.pgc"
+
+}
+
+void init_var()
+{
+ count = 0;
+ memset(utext_var,'a',sizeof(utext_var));
+ memset(utext_array,'a',sizeof(utext_array));
+ memset(p_utext_var,'a',P_VAR_SIZE*sizeof(utext));
+}
+
+//simple select into utext var
+void test_var_1()
+{
+ test("test_var_1: simple select into utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 276 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 276 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 276 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 2", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 281 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 281 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 281 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 286 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 286 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 286 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 4", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 291 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 291 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 291 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+}
+
+
+//simple select using utext var
+void test_var_2()
+{
+ test("test_var_2: simple select using utext var");
+ init_var();
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 305 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 305 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 305 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 306 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 306 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 306 "utext.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 310 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 310 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 310 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 311 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 311 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 311 "utext.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using utext var
+void test_var_3()
+{
+ test("test_var_3: simple update using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 321 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 321 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 321 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 2",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 322 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 322 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 322 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 3",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 323 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 323 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 323 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 324 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 324 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 324 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext var
+void test_var_4()
+{
+ test("test_var_4 : simple delete using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 336 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 336 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 336 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 337 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 337 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 337 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 338 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 338 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 338 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 341 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 341 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 341 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 342 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 342 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 342 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 343 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 343 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 343 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext var
+void test_var_5()
+{
+ test("test_var_5 : simple insert using utext");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 355 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 355 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 355 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 11 )",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 356 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 356 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 356 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 357 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 357 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 357 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 361 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 361 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 361 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 362 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 362 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 362 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 363 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 363 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 363 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext var
+void test_var_6()
+{
+ test("test_var_6 : prepared select into utext var");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=?");
+#line 375 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 375 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 375 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 377 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 377 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 377 "utext.pgc"
+
+ print_utext(utext_var);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 380 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 380 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 380 "utext.pgc"
+
+ print_utext(utext_var);
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 383 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 383 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 383 "utext.pgc"
+
+}
+
+//prepared select using utext var
+void test_var_7()
+{
+ test("test_var_7 : prepared select using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 392 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 392 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 392 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 394 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 394 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 394 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 395 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 395 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 395 "utext.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 398 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 398 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 398 "utext.pgc"
+
+}
+
+//prepared update using utext var
+void test_var_8()
+{
+ test("test_var_8 : prepared update using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 407 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 407 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 407 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 409 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 409 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 409 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 410 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 410 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 410 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 411 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 411 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 411 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 412 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 412 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 412 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 415 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 415 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 415 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using utext var
+void test_var_9()
+{
+ test("test_var_9 : prepared delete using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 426 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 426 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 426 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 428 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 428 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 428 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 429 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 429 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 429 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 430 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 430 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 430 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 433 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 433 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 433 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using utext var
+void test_var_10()
+{
+ test("test_var_10 : prepared insert using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 444 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 444 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 444 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, 13)");
+#line 446 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 446 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 446 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 447 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 447 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 447 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 448 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 448 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 448 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 451 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 451 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 451 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using utext var
+void test_var_11()
+{
+ test("test_var_11 : Open cursor using utext var");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 462 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 462 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 462 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 463 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 463 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 463 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 465 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 465 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 465 "utext.pgc"
+
+ /* declare cursor_var_11 cursor for $1 */
+#line 466 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_var_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 467 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 467 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 467 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_var_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 468 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 468 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 468 "utext.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_var_11", ECPGt_EOIT, ECPGt_EORT);
+#line 470 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 470 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 470 "utext.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 471 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 471 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 471 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 472 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 472 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 472 "utext.pgc"
+
+}
+
+//Fecth cursor into utext var
+void test_var_12()
+{
+ test("test_var_12 : Fecth cursor into utext var");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 481 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 481 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 481 "utext.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=1");
+#line 482 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 482 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 482 "utext.pgc"
+
+ /* declare cursor_var_12 cursor for $1 */
+#line 483 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_var_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 484 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 484 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 484 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_var_12", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 485 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 485 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 485 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_var_12", ECPGt_EOIT, ECPGt_EORT);
+#line 486 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 486 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 486 "utext.pgc"
+
+
+ print_utext(utext_var);
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 490 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 490 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 490 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 491 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 491 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 491 "utext.pgc"
+
+}
+
+//simple insert utext var with L string inited
+void test_var_13()
+{
+/* exec sql begin declare section */
+
+
+#line 498 "utext.pgc"
+ utext utext_local_var [] = L"足球篮球羽毛球" ;
+/* exec sql end declare section */
+#line 499 "utext.pgc"
+
+ test("test_var_13 : simple insert utext var with L string inited");
+ init_var();
+
+ print_utext_str(utext_local_var);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 16 )",
+ ECPGt_utext,(utext_local_var),(long)sizeof(L"足球篮球羽毛球")/4,(long)1,(sizeof(L"足球篮球羽毛球")/4)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 505 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 505 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 505 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 16", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 507 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 507 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 507 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+ init_table_value();
+}
+
+//Open cursor using utext var directly in WHERE Clause
+void test_var_14()
+{
+/* exec sql begin declare section */
+
+
+#line 518 "utext.pgc"
+ utext utext_local_var [ 20 + 1 ] ;
+/* exec sql end declare section */
+#line 519 "utext.pgc"
+
+ memset(utext_local_var,'a',sizeof(utext_local_var));
+
+ test("test_var_14 : Open cursor using utext var directly in WHERE Clause");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 525 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 525 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 525 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 526 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 526 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 526 "utext.pgc"
+
+ ECPGset_var( 0, ( utext_local_var ), __LINE__);\
+ /* declare cursor_var_14 cursor for select count from tb1 where Item = $1 */
+#line 527 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_var_14 cursor for select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 528 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 528 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 528 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_var_14", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 529 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 529 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 529 "utext.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_var_14", ECPGt_EOIT, ECPGt_EORT);
+#line 532 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 532 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 532 "utext.pgc"
+
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 534 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 534 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 534 "utext.pgc"
+
+}
+
+//Test utext_var working with NULL without indicator
+void test_var_15()
+{
+/* exec sql begin declare section */
+
+
+#line 541 "utext.pgc"
+ utext utext_local_var [ 20 + 1 ] ;
+/* exec sql end declare section */
+#line 542 "utext.pgc"
+
+ test("test_var_15 : Test utext_var working with NULL without indicator");
+
+ memset(utext_local_var,'a',sizeof(utext_local_var));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 8", ECPGt_EOIT,
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 546 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 546 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 546 "utext.pgc"
+
+ print_utext(utext_local_var);
+
+ memset(utext_local_var,0x00,sizeof(utext_local_var));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 550 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 550 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 550 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 551 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 551 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 551 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 552 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 552 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 552 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 553 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 553 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 553 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext_var working with NULL with indicator
+void test_var_16()
+{
+/* exec sql begin declare section */
+
+
+
+#line 563 "utext.pgc"
+ utext utext_local_var [ 20 + 1 ] ;
+
+#line 564 "utext.pgc"
+ int utext_var_ind = - 1 ;
+/* exec sql end declare section */
+#line 565 "utext.pgc"
+
+ test("test_var_16 : Test utext_var working with NULL with indicator");
+
+ memset(utext_local_var,'a',sizeof(utext_local_var));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 8", ECPGt_EOIT,
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 569 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 569 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 569 "utext.pgc"
+
+ print_utext(utext_local_var);
+ print_utext_ind(utext_var_ind);
+
+ memset(utext_local_var,0x00,sizeof(utext_local_var));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 574 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 574 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 574 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 575 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 575 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 575 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,(utext_local_var),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 576 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 576 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 576 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 577 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 577 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 577 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+
+//simple select into utext array
+void test_array_1()
+{
+ test("test_array_1 : simple select into utext array ");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 590 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 590 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 590 "utext.pgc"
+
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 595 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 595 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 595 "utext.pgc"
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+ init_var();
+}
+
+//simple select using utext array
+void test_array_2()
+{
+ test("test_array_2 : simple select using array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 606 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 606 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 606 "utext.pgc"
+
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 609 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 609 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 609 "utext.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 613 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 613 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 613 "utext.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using utext array
+void test_array_3()
+{
+ test("test_array_3 : simple update using utext array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 623 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 623 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 623 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 1",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 625 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 625 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 625 "utext.pgc"
+
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 628 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 628 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 628 "utext.pgc"
+
+ printf ("find %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 632 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 632 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 632 "utext.pgc"
+
+ printf ("find %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext array
+void test_array_4()
+{
+ test("test_array_4 : simple delete using utext array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 644 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 644 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 644 "utext.pgc"
+
+
+ count = 100;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 647 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 647 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 647 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 648 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 648 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 648 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 100;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 652 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 652 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 652 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 653 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 653 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 653 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext array
+void test_array_5()
+{
+ test("test_array_5 : simple insert using utext array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 665 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 665 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 665 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 11 )",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 667 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 667 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 667 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 668 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 668 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 668 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 671 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 671 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 671 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 672 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 672 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 672 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext array
+void test_array_6()
+{
+ test("test_array_6 : prepared select into utext array");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count<=?");
+#line 684 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 684 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 684 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"4",(long)1,(long)1,strlen("4"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 686 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 686 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 686 "utext.pgc"
+
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 690 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 690 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 690 "utext.pgc"
+
+}
+
+//prepared select using utext array
+void test_array_7()
+{
+ test("test_array_7 : prepared select using utext array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 698 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 698 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 698 "utext.pgc"
+
+
+ count = 0;
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 701 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 701 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 701 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 702 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 702 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 702 "utext.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 705 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 705 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 705 "utext.pgc"
+
+}
+
+//prepared update using utext array
+void test_array_8()
+{
+ test("test_array_8 : prepared update using utext array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 713 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 713 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 713 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 715 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 715 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 715 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 716 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 716 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 716 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 717 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 717 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 717 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 718 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 718 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 718 "utext.pgc"
+
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 721 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 721 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 721 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using utext array
+void test_array_9()
+{
+ test("test_array_9 : prepared delete using utext array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 731 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 731 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 731 "utext.pgc"
+
+
+ count = 0;
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 734 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 734 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 734 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 735 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 735 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 735 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 736 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 736 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 736 "utext.pgc"
+
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 739 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 739 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 739 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using utext array
+void test_array_10()
+{
+ test("test_array_10 : prepared insert using utext array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 749 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 749 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 749 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, ?)");
+#line 751 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 751 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 751 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"13",(long)2,(long)1,strlen("13"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 752 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 752 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 752 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"15",(long)2,(long)1,strlen("15"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 753 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 753 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 753 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 754 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 754 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 754 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 757 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 757 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 757 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using utext array
+void test_array_11()
+{
+ test("test_array_11 : Open cursor using utext array");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 768 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 768 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 768 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 769 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 769 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 769 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 771 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 771 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 771 "utext.pgc"
+
+ /* declare cursor_array_11 cursor for $1 */
+#line 772 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_array_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 773 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 773 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 773 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_array_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 774 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 774 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 774 "utext.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_array_11", ECPGt_EOIT, ECPGt_EORT);
+#line 776 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 776 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 776 "utext.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 777 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 777 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 777 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 778 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 778 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 778 "utext.pgc"
+
+}
+
+//Fecth cursor into utext array
+void test_array_12()
+{
+ test("test_array_12 : Fecth cursor into utext array");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 787 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 787 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 787 "utext.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count<=3");
+#line 788 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 788 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 788 "utext.pgc"
+
+ /* declare cursor_array_12 cursor for $1 */
+#line 789 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_array_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 790 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 790 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 790 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 791 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 791 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 791 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_utext,(utext_array[1]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 792 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 792 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 792 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 793 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 793 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 793 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_array_12", ECPGt_EOIT, ECPGt_EORT);
+#line 794 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 794 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 794 "utext.pgc"
+
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 798 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 798 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 798 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 799 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 799 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 799 "utext.pgc"
+
+}
+
+//Insert array with L string inited
+void test_array_13()
+{
+/* exec sql begin declare section */
+
+
+#line 806 "utext.pgc"
+ utext utext_array13 [ 4 ] [ VAR_SIZE ] = { L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋" , L"足球篮球羽毛球乒乓球橄榄球棒球冰球" , L"世界杯每隔四年就会举行一次" } ;
+/* exec sql end declare section */
+#line 807 "utext.pgc"
+
+
+ test("test_array_13 : Insert array with L string inited");
+ init_var();
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,(utext_array13[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 813 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 813 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 813 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 21 )",
+ ECPGt_utext,(utext_array13[1]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 814 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 814 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 814 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 22 )",
+ ECPGt_utext,(utext_array13[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 815 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 815 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 815 "utext.pgc"
+
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 22 and count >= 20", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 818 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 818 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 818 "utext.pgc"
+
+ print_array(utext_array,3,VAR_SIZE);
+ init_table_value();
+}
+
+//Open cursor using utext array directly in WHERE Clause
+void test_array_14()
+{
+/* exec sql begin declare section */
+
+
+#line 827 "utext.pgc"
+ utext utext_local_array [ 4 ] [ 20 + 1 ] ;
+/* exec sql end declare section */
+#line 828 "utext.pgc"
+
+
+ test("test_array_14 : Open cursor using utext array directly in WHERE Clause");
+ memset(utext_local_array,'a',sizeof(utext_local_array));
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 833 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 833 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 833 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_local_array),(long)20 + 1,(long)4,(20 + 1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 834 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 834 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 834 "utext.pgc"
+
+
+ ECPGset_var( 1, ( utext_local_array[2] ), __LINE__);\
+ /* declare cursor_array_14 cursor for select count from tb1 where Item = $1 */
+#line 836 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_array_14 cursor for select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_local_array[2]),(long)20 + 1,(long)1,(20 + 1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 837 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 837 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 837 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_array_14", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 838 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 838 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 838 "utext.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_array_14", ECPGt_EOIT, ECPGt_EORT);
+#line 840 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 840 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 840 "utext.pgc"
+
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 842 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 842 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 842 "utext.pgc"
+
+}
+
+
+//utext array working with NULL without using indicator
+void test_array_15()
+{
+/* exec sql begin declare section */
+
+
+#line 850 "utext.pgc"
+ utext utext_local_array [ 3 ] [ 20 ] ;
+/* exec sql end declare section */
+#line 851 "utext.pgc"
+
+ test("test_array_15 : Test utext array working with NULL without using indicator");
+ memset(utext_local_array,'a',sizeof(utext_local_array));
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count >= 8 and count <= 10", ECPGt_EOIT,
+ ECPGt_utext,(utext_local_array),(long)20,(long)3,(20)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 855 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 855 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 855 "utext.pgc"
+
+
+ print_array((void**)utext_local_array,3,20);
+
+ memset(utext_local_array,0x00,sizeof(utext_local_array));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,(utext_local_array[0]),(long)20,(long)1,(20)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 860 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 860 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 860 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,(utext_local_array[1]),(long)20,(long)1,(20)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 861 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 861 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 861 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,(utext_local_array[2]),(long)20,(long)1,(20)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 862 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 862 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 862 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 863 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 863 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 863 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext array working with NULL using indicator
+void test_array_16()
+{
+/* exec sql begin declare section */
+
+
+
+#line 873 "utext.pgc"
+ utext utext_local_array [ 3 ] [ 20 ] ;
+
+#line 874 "utext.pgc"
+ int utext_array_ind [ 3 ] ;
+/* exec sql end declare section */
+#line 875 "utext.pgc"
+
+ test("test_array_16 : Test utext array working with NULL using indicator");
+ memset(utext_local_array,'a',sizeof(utext_local_array));
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count >= 8 and count <= 10", ECPGt_EOIT,
+ ECPGt_utext,(utext_local_array),(long)20,(long)3,(20)*sizeof(utext),
+ ECPGt_int,(utext_array_ind),(long)1,(long)3,sizeof(int), ECPGt_EORT);
+#line 879 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 879 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 879 "utext.pgc"
+
+
+ print_array((void**)utext_local_array,3,20);
+ print_utext_ind(utext_array_ind[0]);
+ print_utext_ind(utext_array_ind[1]);
+ print_utext_ind(utext_array_ind[2]);
+
+ memset(utext_local_array,0x00,sizeof(utext_local_array));
+ utext_array_ind[0]=-1;
+ utext_array_ind[1]=-1;
+ utext_array_ind[2]=-1;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,(utext_local_array[0]),(long)20,(long)1,(20)*sizeof(utext),
+ ECPGt_int,&(utext_array_ind[0]),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 890 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 890 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 890 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,(utext_local_array[1]),(long)20,(long)1,(20)*sizeof(utext),
+ ECPGt_int,&(utext_array_ind[1]),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 891 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 891 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 891 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,(utext_local_array[2]),(long)20,(long)1,(20)*sizeof(utext),
+ ECPGt_int,&(utext_array_ind[2]),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 892 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 892 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 892 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 893 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 893 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 893 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+
+//simple select into utext pointer
+void test_pvar_1()
+{
+ test("test_pvar_1 : simple select into utext pointer");
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 904 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 904 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 904 "utext.pgc"
+
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 2", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 909 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 909 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 909 "utext.pgc"
+
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 914 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 914 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 914 "utext.pgc"
+
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 4", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 919 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 919 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 919 "utext.pgc"
+
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ init_table_value();
+}
+
+
+//simple select using utext pointer
+void test_pvar_2()
+{
+ test("test_pvar_2 : simple select using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 934 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 934 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 934 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 935 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 935 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 935 "utext.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 939 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 939 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 939 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 940 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 940 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 940 "utext.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+
+//simple update using utext pointer
+void test_pvar_3()
+{
+ test("test_pvar_3 : simple update using utext pointer");
+ init_var();
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 952 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 952 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 952 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 2",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 953 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 953 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 953 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 3",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 954 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 954 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 954 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 955 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 955 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 955 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext pointer
+void test_pvar_4()
+{
+ test("test_pvar_4 : simple delete using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 967 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 967 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 967 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 968 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 968 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 968 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 969 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 969 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 969 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 973 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 973 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 973 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 974 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 974 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 974 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 975 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 975 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 975 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext pointer
+void test_pvar_5()
+{
+ test("test_pvar_5 : simple insert using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 987 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 987 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 987 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 988 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 988 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 988 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 990 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 990 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 990 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext pointer
+void test_pvar_6()
+{
+ test("test_pvar_6 : prepared select into utext pointer");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=?");
+#line 1002 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1002 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1002 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1004 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1004 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1004 "utext.pgc"
+
+ print_utext(p_utext_var);
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1008 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1008 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1008 "utext.pgc"
+
+ print_utext(p_utext_var);
+ init_table_value();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 1012 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1012 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1012 "utext.pgc"
+
+
+}
+
+//prepared select using utext pointer
+void test_pvar_7()
+{
+ test("test_pvar_7 : prepared select using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1022 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1022 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1022 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 1024 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1024 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1024 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1025 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1025 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1025 "utext.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 1028 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1028 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1028 "utext.pgc"
+
+}
+
+//prepared update using utext pointer
+void test_pvar_8()
+{
+ test("test_pvar_8 : prepared update using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1037 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1037 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1037 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 1039 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1039 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1039 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1040 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1040 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1040 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1041 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1041 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1041 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1042 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1042 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1042 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 1045 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1045 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1045 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using utext pointer
+void test_pvar_9()
+{
+ test("test_pvar_9 : prepared delete using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1056 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1056 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1056 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 1058 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1058 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1058 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1059 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1059 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1059 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1060 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1060 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1060 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 1063 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1063 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1063 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using utext pointer
+void test_pvar_10()
+{
+ test("test_pvar_10 : prepared insert using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1074 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1074 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1074 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, 13)");
+#line 1076 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1076 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1076 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1077 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1077 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1077 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1078 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1078 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1078 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 1081 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1081 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1081 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using utext pointer
+void test_pvar_11()
+{
+ test("test_pvar_11 : Open cursor using utext pointer");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 1092 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1092 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1092 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1093 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1093 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1093 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 1095 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1095 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1095 "utext.pgc"
+
+ /* declare cursor_pvar_11 cursor for $1 */
+#line 1096 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_pvar_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1097 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1097 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1097 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_pvar_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1098 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1098 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1098 "utext.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_pvar_11", ECPGt_EOIT, ECPGt_EORT);
+#line 1100 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1100 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1100 "utext.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 1101 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1101 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1101 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 1102 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1102 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1102 "utext.pgc"
+
+}
+
+//Fecth cursor into utext pointer
+void test_pvar_12()
+{
+ test("test_pvar_12 : Fecth cursor into utext pointer");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 1111 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1111 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1111 "utext.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=1");
+#line 1112 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1112 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1112 "utext.pgc"
+
+ /* declare cursor_pvar_12 cursor for $1 */
+#line 1113 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_pvar_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1114 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1114 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1114 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_pvar_12", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1115 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1115 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1115 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_pvar_12", ECPGt_EOIT, ECPGt_EORT);
+#line 1116 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1116 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1116 "utext.pgc"
+
+
+ print_utext(p_utext_var);
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 1120 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1120 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1120 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 1121 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1121 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1121 "utext.pgc"
+
+}
+
+//simple insert utext pointer with L string inited
+void test_pvar_13()
+{
+/* exec sql begin declare section */
+
+
+#line 1128 "utext.pgc"
+ utext * utext_local_pvar = L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋" ;
+/* exec sql end declare section */
+#line 1129 "utext.pgc"
+
+ test("test_pvar_13 : simple insert utext pointer with L string inited");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 16 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1133 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1133 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1133 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 16", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 1135 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1135 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1135 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+ init_table_value();
+}
+
+//Open cursor using utext var directly in WHERE Clause
+void test_pvar_14()
+{
+/* exec sql begin declare section */
+
+
+#line 1146 "utext.pgc"
+ utext * utext_local_pvar ;
+/* exec sql end declare section */
+#line 1147 "utext.pgc"
+
+ utext_local_pvar = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ memset(utext_local_pvar,'a',P_VAR_SIZE*sizeof(utext));
+
+ test("test_pvar_14 : Open cursor using utext var directly in WHERE Clause");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 1154 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1154 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1154 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1155 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1155 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1155 "utext.pgc"
+
+
+ ECPGset_var( 2, &( utext_local_pvar ), __LINE__);\
+ /* declare cursor_pvar_14 cursor for select count from tb1 where Item = $1 */
+#line 1157 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_pvar_14 cursor for select count from tb1 where Item = $1 ",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1158 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1158 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1158 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_pvar_14", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1159 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1159 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1159 "utext.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_pvar_14", ECPGt_EOIT, ECPGt_EORT);
+#line 1161 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1161 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1161 "utext.pgc"
+
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 1163 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1163 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1163 "utext.pgc"
+
+}
+
+
+//utext pointer working with NULL without using indicator
+void test_pvar_15()
+{
+/* exec sql begin declare section */
+
+
+#line 1171 "utext.pgc"
+ utext * utext_local_pvar ;
+/* exec sql end declare section */
+#line 1172 "utext.pgc"
+
+ utext_local_pvar = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ memset(utext_local_pvar,'a',P_VAR_SIZE*sizeof(utext));
+
+ test("test_pvar_15 : Test utext pointer working with NULL without using indicator");
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 8", ECPGt_EOIT,
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1178 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1178 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1178 "utext.pgc"
+
+
+ print_utext(utext_local_pvar);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1182 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1182 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1182 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1183 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1183 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1183 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1184 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1184 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1184 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1185 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1185 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1185 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext pointer working with NULL using indicator
+void test_pvar_16()
+{
+/* exec sql begin declare section */
+
+
+
+#line 1195 "utext.pgc"
+ utext * utext_local_pvar ;
+
+#line 1196 "utext.pgc"
+ int utext_pvar_ind = - 1 ;
+/* exec sql end declare section */
+#line 1197 "utext.pgc"
+
+ utext_local_pvar = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ memset(utext_local_pvar,'a',P_VAR_SIZE*sizeof(utext));
+
+ test("test_pvar_16 : Test utext pointer working with NULL using indicator");
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 8", ECPGt_EOIT,
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_pvar_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 1203 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1203 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1203 "utext.pgc"
+
+
+ print_utext(utext_local_pvar);
+ print_utext_ind(utext_pvar_ind);
+
+ memset(utext_local_pvar,0,P_VAR_SIZE*sizeof(utext));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_pvar_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 1209 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1209 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1209 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_pvar_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 1210 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1210 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1210 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_pvar_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 1211 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1211 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1211 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1212 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1212 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1212 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext uninitialized pointer getting and setting data
+void test_pvar_17()
+{
+/* exec sql begin declare section */
+
+
+
+#line 1222 "utext.pgc"
+ utext * utext_local_pvar = 0 ;
+
+#line 1223 "utext.pgc"
+ char local_str [ 50 ] ;
+/* exec sql end declare section */
+#line 1224 "utext.pgc"
+
+
+ test("test_pvar_17 : Test utext uninitialized pointer getting and setting data");
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1228 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1228 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1228 "utext.pgc"
+
+
+ print_utext_str(utext_local_pvar);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1232 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1232 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1232 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 18", ECPGt_EOIT,
+ ECPGt_char,(local_str),(long)50,(long)1,(50)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1233 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1233 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1233 "utext.pgc"
+
+ printf ("Find Item = %s where Count=18\n", local_str);
+
+ init_table_value();
+}
+
+//utext uninitialized pointer working with NULL without using indicator
+void test_pvar_18()
+{
+/* exec sql begin declare section */
+
+
+#line 1243 "utext.pgc"
+ utext * utext_local_pvar = 0 ;
+/* exec sql end declare section */
+#line 1244 "utext.pgc"
+
+
+ test("test_pvar_18 : Test utext uninitialized pointer working with NULL without using indicator");
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 8", ECPGt_EOIT,
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1248 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1248 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1248 "utext.pgc"
+
+
+ print_utext_size(utext_local_pvar,1);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1252 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1252 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1252 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1253 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1253 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1253 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1254 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1254 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1254 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1255 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1255 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1255 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext uninitialized pointer working with NULL using indicator
+void test_pvar_19()
+{
+/* exec sql begin declare section */
+
+
+
+#line 1265 "utext.pgc"
+ utext * utext_local_pvar = 0 ;
+
+#line 1266 "utext.pgc"
+ int utext_pvar_ind = - 1 ;
+/* exec sql end declare section */
+#line 1267 "utext.pgc"
+
+
+ test("test_pvar_19 : Test utext uninitialized pointer working with NULL using indicator");
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 8", ECPGt_EOIT,
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_pvar_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 1271 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1271 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1271 "utext.pgc"
+
+
+ print_utext_size(utext_local_pvar,1);
+ print_utext_ind(utext_pvar_ind);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_pvar_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 1276 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1276 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1276 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_pvar_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 1277 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1277 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1277 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_int,&(utext_pvar_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 1278 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1278 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1278 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1279 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1279 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1279 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//Test select into host varaibles from descriptor
+void test_desc_1()
+{
+ /* exec sql begin declare section */
+
+
+
+
+
+#line 1289 "utext.pgc"
+ utext utext_local_var [ 20 ] ;
+
+#line 1290 "utext.pgc"
+ utext * utext_local_pvar = 0 ;
+
+#line 1291 "utext.pgc"
+ utext ** utext_local_ppvar = 0 ;
+
+#line 1292 "utext.pgc"
+ char desc1 [ 8 ] = "outdesc" ;
+/* exec sql end declare section */
+#line 1293 "utext.pgc"
+
+
+// ECPGdebug(1, stderr);
+
+ ECPGallocate_desc(__LINE__, "indesc");
+#line 1297 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1297 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();
+#line 1297 "utext.pgc"
+
+ ECPGallocate_desc(__LINE__, (desc1));
+#line 1298 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1298 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();
+#line 1298 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item , count from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_descriptor, (desc1), 1L, 1L, 1L,
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1300 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1300 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1300 "utext.pgc"
+
+
+ { ECPGget_desc(__LINE__, (desc1), 1,ECPGd_data,
+ ECPGt_utext,(utext_local_var),(long)20,(long)1,(20)*sizeof(utext), ECPGd_EODT);
+
+#line 1302 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1302 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1302 "utext.pgc"
+
+ print_utext(utext_local_var);
+
+ { ECPGget_desc(__LINE__, (desc1), 1,ECPGd_data,
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext), ECPGd_EODT);
+
+#line 1305 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1305 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1305 "utext.pgc"
+
+ print_utext_str(utext_local_pvar);
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item , count from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_descriptor, (desc1), 1L, 1L, 1L,
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1309 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1309 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1309 "utext.pgc"
+
+ { ECPGget_desc(__LINE__, (desc1), 1,ECPGd_data,
+ ECPGt_utext,&(utext_local_ppvar),(long)0,(long)0,(1)*sizeof(utext), ECPGd_EODT);
+
+#line 1310 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1310 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1310 "utext.pgc"
+
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_str(utext_local_ppvar[2]);
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item , count from tb1 where count <= 10 and count >= 8", ECPGt_EOIT,
+ ECPGt_descriptor, (desc1), 1L, 1L, 1L,
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1316 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1316 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1316 "utext.pgc"
+
+ { ECPGget_desc(__LINE__, (desc1), 1,ECPGd_data,
+ ECPGt_utext,&(utext_local_ppvar),(long)0,(long)0,(1)*sizeof(utext), ECPGd_EODT);
+
+#line 1317 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1317 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1317 "utext.pgc"
+
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_str(utext_local_ppvar[2]);
+
+
+ ECPGdeallocate_desc(__LINE__, "indesc");
+#line 1323 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1323 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();
+#line 1323 "utext.pgc"
+
+ ECPGdeallocate_desc(__LINE__, (desc1));
+#line 1324 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1324 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();
+#line 1324 "utext.pgc"
+
+}
+
+
+//Test uninitialized pointer to pointer utext_var without initialize
+void test_pp_var_1()
+{
+/* exec sql begin declare section */
+
+
+
+
+#line 1332 "utext.pgc"
+ utext ** utext_local_ppvar = 0 ;
+
+#line 1333 "utext.pgc"
+ int * utext_local_ppvar_ind = 0 ;
+
+#line 1334 "utext.pgc"
+ char utext_local_char [ 80 ] ;
+/* exec sql end declare section */
+#line 1335 "utext.pgc"
+
+ test("test_pp_var_1 : Test pointer to pointer utext_var without initialize");
+
+ //memset(utext_local_var,'a',sizeof(utext_local_var));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,&(utext_local_ppvar),(long)0,(long)0,(1)*sizeof(utext),
+ ECPGt_int,&(utext_local_ppvar_ind),(long)1,(long)0,sizeof(int), ECPGt_EORT);
+#line 1339 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1339 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1339 "utext.pgc"
+
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_ind(utext_local_ppvar_ind[0]);
+
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_ind(utext_local_ppvar_ind[1]);
+
+ print_utext_str(utext_local_ppvar[2]);
+ print_utext_ind(utext_local_ppvar_ind[2]);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 28 )",
+ ECPGt_utext,&(utext_local_ppvar[1]),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1349 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1349 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1349 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 28", ECPGt_EOIT,
+ ECPGt_char,(utext_local_char),(long)80,(long)1,(80)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1350 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1350 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1350 "utext.pgc"
+
+ printf("%s\n",utext_local_char);
+
+ init_table_value();
+}
+
+
+//Test uninitialized pointer to pointer utext_var working with NULL without indicator
+void test_pp_var_2()
+{
+/* exec sql begin declare section */
+
+
+#line 1361 "utext.pgc"
+ utext ** utext_local_ppvar = 0 ;
+/* exec sql end declare section */
+#line 1362 "utext.pgc"
+
+ test("test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator");
+
+ //memset(utext_local_var,'a',sizeof(utext_local_var));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count >= 8 and count <= 10", ECPGt_EOIT,
+ ECPGt_utext,&(utext_local_ppvar),(long)0,(long)0,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1366 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1366 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1366 "utext.pgc"
+
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_str(utext_local_ppvar[2]);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,&(utext_local_ppvar[0]),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1371 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1371 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1371 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,&(utext_local_ppvar[1]),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1372 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1372 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1372 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,&(utext_local_ppvar[2]),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1373 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1373 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1373 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1374 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1374 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1374 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//Test uninitialized pointer to pointer utext_var working with NULL with indicator
+void test_pp_var_3()
+{
+/* exec sql begin declare section */
+
+
+
+#line 1384 "utext.pgc"
+ utext ** utext_local_ppvar = 0 ;
+
+#line 1385 "utext.pgc"
+ int * utext_local_ppvar_ind = 0 ;
+/* exec sql end declare section */
+#line 1386 "utext.pgc"
+
+ test("test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator");
+
+ //memset(utext_local_var,'a',sizeof(utext_local_var));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count >= 8 and count <= 10", ECPGt_EOIT,
+ ECPGt_utext,&(utext_local_ppvar),(long)0,(long)0,(1)*sizeof(utext),
+ ECPGt_int,&(utext_local_ppvar_ind),(long)1,(long)0,sizeof(int), ECPGt_EORT);
+#line 1390 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1390 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1390 "utext.pgc"
+
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_ind(utext_local_ppvar_ind[0]);
+
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_ind(utext_local_ppvar_ind[1]);
+
+ print_utext_str(utext_local_ppvar[2]);
+ print_utext_ind(utext_local_ppvar_ind[2]);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_utext,&(utext_local_ppvar[0]),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1400 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1400 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1400 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_utext,&(utext_local_ppvar[1]),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1401 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1401 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1401 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,&(utext_local_ppvar[2]),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 1402 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1402 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1402 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 1403 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 1403 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 1403 "utext.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+void test_all()
+{
+ test_var_1();
+ test_var_2();
+ test_var_3();
+ test_var_4();
+ test_var_5();
+ test_var_6();
+ test_var_7();
+ test_var_8();
+ test_var_9();
+ test_var_10();
+ test_var_11();
+ test_var_12();
+ test_var_13();
+ test_var_14();
+ test_var_15();
+ test_var_16();
+ test_array_1();
+ test_array_2();
+ test_array_3();
+ test_array_4();
+ test_array_5();
+ test_array_6();
+ test_array_7();
+ test_array_8();
+ test_array_9();
+ test_array_10();
+ test_array_11();
+ test_array_12();
+ test_array_13();
+ test_array_14();
+ test_array_15();
+ test_array_16();
+ test_pvar_1();
+ test_pvar_2();
+ test_pvar_3();
+ test_pvar_4();
+ test_pvar_5();
+ test_pvar_6();
+ test_pvar_7();
+ test_pvar_8();
+ test_pvar_9();
+ test_pvar_10();
+ test_pvar_11();
+ test_pvar_12();
+ test_pvar_13();
+ test_pvar_14();
+ test_pvar_15();
+ test_pvar_16();
+ test_pvar_17();
+ test_pvar_18();
+ test_pvar_19();
+ test_pp_var_1();
+ test_pp_var_2();
+ test_pp_var_3();
+}
+
+int main(int argc, char *argv[])
+{
+// ECPGdebug(1, stderr);
+ if(test_init() !=0)
+ return -1;
+
+ test_all();
+ test_finish();
+
+ return 0;
+}
diff --git a/src/interfaces/ecpg/test/expected/sql-utext.stderr b/src/interfaces/ecpg/test/expected/sql-utext.stderr
new file mode 100644
index 0000000..885cfe4
--- /dev/null
+++ b/src/interfaces/ecpg/test/expected/sql-utext.stderr
@@ -0,0 +1,2 @@
+SQL error:
+SQL error:
diff --git a/src/interfaces/ecpg/test/expected/sql-utext.stdout b/src/interfaces/ecpg/test/expected/sql-utext.stdout
new file mode 100644
index 0000000..c8dfd5b
--- /dev/null
+++ b/src/interfaces/ecpg/test/expected/sql-utext.stdout
@@ -0,0 +1,433 @@
+
+test_var_1: simple select into utext var
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E9A 0x00006D32 0x00006B27 0x00006D32 0x0000975E 0x00006D32 0x00005927 0x00006D0B
+0x00006D32 0x00005317 0x00007F8E 0x00006D32 0x00005357 0x00007F8E 0x00006D32 0x00005357
+0x00006781 0x00006D32 0x00006CA1 0x00006709
+======End utext_var content======
+utext_var_ind = 23
+
+test_var_2: simple select using utext var
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_3: simple update using utext var
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_var_4 : simple delete using utext var
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_5 : simple insert using utext
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_6 : prepared select into utext var
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+
+test_var_7 : prepared select using utext var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_8 : prepared update using utext var
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_9 : prepared delete using utext var
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_10 : prepared insert using utext var
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_11 : Open cursor using utext var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_12 : Fecth cursor into utext var
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+
+test_var_13 : simple insert utext var with L string inited
+======print utext_var_str content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403
+======End utext_var_str content======
+======print utext_var content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+
+test_var_14 : Open cursor using utext var directly in WHERE Clause
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_15 : Test utext_var working with NULL without indicator
+======print utext_var content======
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+found 6 rows for Item being NULL
+
+test_var_16 : Test utext_var working with NULL with indicator
+======print utext_var content======
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_array_1 : simple select into utext array
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+---->array[3]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[2]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[3]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_2 : simple select using array
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_3 : simple update using utext array
+find 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+find 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_4 : simple delete using utext array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_5 : simple insert using utext array
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_6 : prepared select into utext array
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+---->array[3]:
+0x00004E9A 0x00006D32 0x00006B27 0x00006D32 0x0000975E 0x00006D32 0x00005927 0x00006D0B
+0x00006D32 0x00005317 0x00007F8E 0x00006D32 0x00005357 0x00007F8E 0x00006D32 0x00005357
+0x00006781 0x00006D32 0x00006CA1 0x00006709
+
+
+test_array_7 : prepared select using utext array
+count=1 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_8 : prepared update using utext array
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_9 : prepared delete using utext array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_10 : prepared insert using utext array
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_11 : Open cursor using utext array
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_12 : Fecth cursor into utext array
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+---->array[3]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_13 : Insert array with L string inited
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00000000 0x00000000 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+
+
+test_array_14 : Open cursor using utext array directly in WHERE Clause
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_15 : Test utext array working with NULL without using indicator
+---->array[0]:
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[1]:
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[2]:
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+found 6 rows for Item being NULL
+
+test_array_16 : Test utext array working with NULL using indicator
+---->array[0]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[1]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[2]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+utext_var_ind = -1
+utext_var_ind = -1
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_pvar_1 : simple select into utext pointer
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E9A 0x00006D32 0x00006B27 0x00006D32 0x0000975E 0x00006D32 0x00005927 0x00006D0B
+0x00006D32 0x00005317 0x00007F8E 0x00006D32 0x00005357 0x00007F8E 0x00006D32 0x00005357
+0x00006781 0x00006D32 0x00006CA1 0x00006709
+======End utext_var content======
+utext_var_ind = 0
+
+test_pvar_2 : simple select using utext pointer
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=0 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_3 : simple update using utext pointer
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_pvar_4 : simple delete using utext pointer
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_5 : simple insert using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_6 : prepared select into utext pointer
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+
+test_pvar_7 : prepared select using utext pointer
+count=0 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_8 : prepared update using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_9 : prepared delete using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_10 : prepared insert using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_11 : Open cursor using utext pointer
+count=0 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_12 : Fecth cursor into utext pointer
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+
+test_pvar_13 : simple insert utext pointer with L string inited
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+
+test_pvar_14 : Open cursor using utext var directly in WHERE Clause
+count=0 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_15 : Test utext pointer working with NULL without using indicator
+======print utext_var content======
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+found 6 rows for Item being NULL
+
+test_pvar_16 : Test utext pointer working with NULL using indicator
+======print utext_var content======
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_pvar_17 : Test utext uninitialized pointer getting and setting data
+======print utext_var_str content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B
+======End utext_var_str content======
+Find Item = 太𠮷𠜱平洋𠱓大西洋印度洋北冰洋 where Count=18
+
+test_pvar_18 : Test utext uninitialized pointer working with NULL without using indicator
+======print utext_var content======
+0x00000000
+======End utext_var content======
+found 6 rows for Item being NULL
+
+test_pvar_19 : Test utext uninitialized pointer working with NULL using indicator
+======print utext_var content======
+0x00000000
+======End utext_var content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_pp_var_1 : Test pointer to pointer utext_var without initialize
+======print utext_var_str content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B
+======End utext_var_str content======
+utext_var_ind = 0
+======print utext_var_str content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403
+======End utext_var_str content======
+utext_var_ind = 0
+======print utext_var_str content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F
+======End utext_var_str content======
+utext_var_ind = 0
+足球篮球羽毛球乒乓球橄榄球棒球冰球
+
+test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator
+======print utext_var_str content======
+
+======End utext_var_str content======
+======print utext_var_str content======
+
+======End utext_var_str content======
+======print utext_var_str content======
+
+======End utext_var_str content======
+found 6 rows for Item being NULL
+
+test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator
+======print utext_var_str content======
+
+======End utext_var_str content======
+utext_var_ind = -1
+======print utext_var_str content======
+
+======End utext_var_str content======
+utext_var_ind = -1
+======print utext_var_str content======
+
+======End utext_var_str content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
diff --git a/src/interfaces/ecpg/test/expected/sql-uvarchar.c b/src/interfaces/ecpg/test/expected/sql-uvarchar.c
new file mode 100644
index 0000000..288eb14
--- /dev/null
+++ b/src/interfaces/ecpg/test/expected/sql-uvarchar.c
@@ -0,0 +1,2580 @@
+/* Processed by ecpg (regression mode) */
+/* These include files are added by the preprocessor */
+#define ECPG_ENABLE_UTEXT 1
+#include <ecpglib.h>
+#include <ecpgerrno.h>
+#include <sqlca.h>
+/* End of automatic include section */
+#define ECPGdebug(X,Y) ECPGdebug((X)+100,(Y))
+
+#line 1 "uvarchar.pgc"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+#line 1 "regression.h"
+
+
+
+
+
+
+#line 5 "uvarchar.pgc"
+
+
+#define test(msg) printf("\n%s\n",msg)
+#define VAR_SIZE 20
+#define ARRAY_SIZE 4
+
+/* Following is UTF8 and UTF16 characters mapping table */
+/*太𠮷𠜱平洋𠱓大西洋印度洋北冰洋
+0x592A,0x5E73,0x6D0B,0x5927,0x897F,0x6D0B,0x5370,0x5EA6,
+0x6D0B,0x5317,0x51B0,0x6D0B,0x548C,0x5357,0x51B0,0x6D0B,
+0x0000,0x0000,0x0000,0x0000*/
+
+/*足球篮球羽毛球乒乓球橄榄球棒球冰球
+0x8DB3,0x7403,0x7BEE,0x7403,0x7FBD,0x6BDB,0x7403,0x4E52,
+0x4E53,0x7403,0x6A44,0x6984,0x7403,0x68D2,0x7403,0x51B0,
+0x7403,0x0000,0x0000,0x0000
+*/
+
+/*世界杯每隔四年就会举行一次每次𠲖个球队
+0x4E16,0x754C,0x676F,0x6BCF,0x9694,0x56DB,0x5E74,0x5C31,
+0x4F1A,0x4E3E,0x884C,0x4E00,0x6B21,0x6BCF,0x6B21,0x0033,
+0x0032,0x4E2A,0x7403,0x961F
+*/
+
+/* 亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲
+0x4E9A,0x6D32,0x6B27,0x6D32,0x975E,0x6D32,0x5927,0x6D0B,
+0x6D32,0x5317,0x7F8E,0x6D32,0x5357,0x7F8E,0x6D32,0x5357,
+0x6781,0x6D32,0x6CA1,0x6709,0x5317,0x6781,0x6D32
+*/
+
+/* exec sql begin declare section */
+
+
+
+
+
+
+
+
+
+#line 36 "uvarchar.pgc"
+ struct uvarchar_1 { int len; utext arr[ VAR_SIZE ]; } uvarchar_var ;
+
+#line 37 "uvarchar.pgc"
+ struct uvarchar_2 { int len; utext arr[ VAR_SIZE ]; } uvarchar_array [ ARRAY_SIZE ] ;
+
+#line 39 "uvarchar.pgc"
+ int uvarchar_var_ind ;
+
+#line 41 "uvarchar.pgc"
+ int count ;
+
+#line 42 "uvarchar.pgc"
+ int count_array [ 4 ] = { 1 , 2 , 3 , 4 } ;
+
+#line 43 "uvarchar.pgc"
+ int total_tuples = 0 ;
+/* exec sql end declare section */
+#line 44 "uvarchar.pgc"
+
+
+void print_uvarchar(void);
+void print_uvarchar_ind(int uvarchar_var_ind);
+void print_local_uvarchar(utext *utext_var, int var_len);
+void print_array(void);
+int test_init(void);
+void test_finish(void);
+void init_table_value(void);
+void init_var(void);
+
+void test_var_1(void);
+void test_var_2(void);
+void test_var_3(void);
+void test_var_4(void);
+void test_var_5(void);
+void test_var_6(void);
+void test_var_7(void);
+void test_var_8(void);
+void test_var_9(void);
+void test_var_10(void);
+void test_var_11(void);
+void test_var_12(void);
+void test_var_13(void);
+void test_var_14(void);
+void test_var_15(void);
+void test_var_16(void);
+void test_array_1(void);
+void test_array_2(void);
+void test_array_3(void);
+void test_array_4(void);
+void test_array_5(void);
+void test_array_6(void);
+void test_array_7(void);
+void test_array_8(void);
+void test_array_9(void);
+void test_array_10(void);
+void test_array_11(void);
+void test_array_12(void);
+void test_array_13(void);
+void test_array_14(void);
+void test_array_15(void);
+void test_array_16(void);
+
+void test_all(void);
+
+void print_uvarchar()
+{
+ int i;
+
+ printf ("---->uvarchar variable,len=%d:\n",uvarchar_var.len);
+ for(i=0; i<VAR_SIZE; i++)
+ {
+ printf ("0x%04X ", uvarchar_var.arr[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_uvarchar_ind(int uvarchar_var_ind)
+{
+ printf("uvarchar_var_ind = %d\n",uvarchar_var_ind);
+}
+
+void print_local_uvarchar(utext *utext_var, int var_len)
+{
+ int i;
+
+ printf ("---->uvarchar variable,len=%d:\n",var_len);
+ for(i=0; i<20; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_array()
+{
+ int i,j;
+
+ for(i=0; i<ARRAY_SIZE; i++)
+ {
+ printf ("---->uvarchar array[%d]:(len=%d)\n", i,uvarchar_array[i].len);
+
+ for(j=0; j<VAR_SIZE; j++)
+ {
+ printf ("0x%04X ", uvarchar_array[i].arr[j]);
+ if(j>6 && (j+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+ }
+
+ printf("\n");
+}
+
+int test_init()
+{
+ { ECPGconnect(__LINE__, 0, "ecpg1_regression" , NULL, NULL , NULL, 0); }
+#line 145 "uvarchar.pgc"
+
+
+ { ECPGsetcommit(__LINE__, "on", NULL);}
+#line 147 "uvarchar.pgc"
+
+ /* exec sql whenever sql_warning sqlprint ; */
+#line 148 "uvarchar.pgc"
+
+ /* exec sql whenever sqlerror sqlprint ; */
+#line 149 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "set client_encoding = 'UTF8'", ECPGt_EOIT, ECPGt_EORT);
+#line 151 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 151 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 151 "uvarchar.pgc"
+
+
+ //initialization of test table
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "create table tb1 ( Item varchar , count integer )", ECPGt_EOIT, ECPGt_EORT);
+#line 154 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 154 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 154 "uvarchar.pgc"
+
+
+ init_table_value();
+
+ return 0;
+}
+
+void test_finish()
+{
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "drop table tb1", ECPGt_EOIT, ECPGt_EORT);
+#line 163 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 163 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 163 "uvarchar.pgc"
+
+ { ECPGdisconnect(__LINE__, "ALL");
+#line 164 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 164 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 164 "uvarchar.pgc"
+
+}
+
+
+void init_table_value()
+{
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "truncate tb1", ECPGt_EOIT, ECPGt_EORT);
+#line 170 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 170 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 170 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋' , 1 )", ECPGt_EOIT, ECPGt_EORT);
+#line 172 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 172 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 172 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( '足球篮球羽毛球乒乓球橄榄球棒球冰球' , 2 )", ECPGt_EOIT, ECPGt_EORT);
+#line 173 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 173 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 173 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( '世界杯每隔四年就会举行一次每次𠲖个球队' , 3 )", ECPGt_EOIT, ECPGt_EORT);
+#line 174 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 174 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 174 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( '亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲' , 4 )", ECPGt_EOIT, ECPGt_EORT);
+#line 175 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 175 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 175 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 ( count ) values ( 8 )", ECPGt_EOIT, ECPGt_EORT);
+#line 177 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 177 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 177 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 ( count ) values ( 9 )", ECPGt_EOIT, ECPGt_EORT);
+#line 178 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 178 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 178 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 ( count ) values ( 10 )", ECPGt_EOIT, ECPGt_EORT);
+#line 179 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 179 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 179 "uvarchar.pgc"
+
+}
+
+void init_var()
+{
+ int i;
+ memset((void*)&uvarchar_var,'a',sizeof(uvarchar_var));
+ uvarchar_var.len = 0;
+
+ memset((char*)uvarchar_array,'a',sizeof(uvarchar_array)*ARRAY_SIZE);
+
+ for(i=0;i<ARRAY_SIZE;i++)
+ {
+ uvarchar_array[i].len = 0;
+ }
+
+ uvarchar_var_ind = 0;
+}
+
+//simple select into uvarchar
+void test_var_1()
+{
+ test("test_var_1 : simple select into uvarchar var");
+ init_var();
+ //print_uvarchar();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 205 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 205 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 205 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 2", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 211 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 211 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 211 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 216 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 216 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 216 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 4", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 221 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 221 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 221 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+}
+
+
+//simple select using uvarchar
+void test_var_2()
+{
+ test("test_var_2 : simple select using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 234 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 234 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 234 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 235 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 235 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 235 "uvarchar.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 238 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 238 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 238 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 239 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 239 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 239 "uvarchar.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using uvarchar
+void test_var_3()
+{
+ test("test_var_3 : simple update using uvarchar");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 249 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 249 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 249 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 2",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 250 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 250 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 250 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 3",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 251 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 251 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 251 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 252 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 252 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 252 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using uvarchar var
+void test_var_4()
+{
+ test("test_var_4 : simple delete using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 264 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 264 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 264 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 265 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 265 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 265 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 266 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 266 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 266 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 269 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 269 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 269 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 270 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 270 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 270 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 271 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 271 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 271 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using uvarchar var
+void test_var_5()
+{
+ test("test_var_5 : simple insert using uvarchar");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 283 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 283 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 283 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 11 )",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 284 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 284 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 284 "uvarchar.pgc"
+
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 287 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 287 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 287 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 288 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 288 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 288 "uvarchar.pgc"
+
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 291 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 291 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 291 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 294 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 294 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 294 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into uvarchar var
+void test_var_6()
+{
+ test("test_var_6 : prepared select into uvarchar var");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=?");
+#line 306 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 306 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 306 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 308 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 308 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 308 "uvarchar.pgc"
+
+ print_uvarchar();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 311 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 311 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 311 "uvarchar.pgc"
+
+ print_uvarchar();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 314 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 314 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 314 "uvarchar.pgc"
+
+}
+
+//prepared select using uvarchar var
+void test_var_7()
+{
+ test("test_var_7 : prepared select using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 323 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 323 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 323 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 325 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 325 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 325 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 326 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 326 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 326 "uvarchar.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 329 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 329 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 329 "uvarchar.pgc"
+
+}
+
+//prepared update using uvarchar var
+void test_var_8()
+{
+ test("test_var_8 : prepared update using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 338 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 338 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 338 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 340 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 340 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 340 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 341 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 341 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 341 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 342 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 342 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 342 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 343 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 343 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 343 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 346 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 346 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 346 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using uvarchar var
+void test_var_9()
+{
+ test("test_var_9 : prepared delete using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 357 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 357 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 357 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 359 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 359 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 359 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 360 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 360 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 360 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 361 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 361 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 361 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 364 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 364 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 364 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using uvarchar var
+void test_var_10()
+{
+ test("test_var_10 : prepared insert using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 375 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 375 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 375 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, 13)");
+#line 377 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 377 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 377 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 378 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 378 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 378 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 379 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 379 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 379 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 382 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 382 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 382 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using uvarchar var
+void test_var_11()
+{
+ test("test_var_11 : Open cursor using uvarchar var");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 393 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 393 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 393 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 394 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 394 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 394 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 396 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 396 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 396 "uvarchar.pgc"
+
+ /* declare cursor_var_11 cursor for $1 */
+#line 397 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_var_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 398 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 398 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 398 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_var_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 399 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 399 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 399 "uvarchar.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_var_11", ECPGt_EOIT, ECPGt_EORT);
+#line 401 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 401 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 401 "uvarchar.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 402 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 402 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 402 "uvarchar.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 403 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 403 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 403 "uvarchar.pgc"
+
+}
+
+//Fecth cursor into uvarchar var
+void test_var_12()
+{
+ test("test_var_12 : Fecth cursor into uvarchar var");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 412 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 412 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 412 "uvarchar.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=1");
+#line 413 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 413 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 413 "uvarchar.pgc"
+
+ /* declare cursor_var_12 cursor for $1 */
+#line 414 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_var_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 415 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 415 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 415 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_var_12", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 416 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 416 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 416 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_var_12", ECPGt_EOIT, ECPGt_EORT);
+#line 417 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 417 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 417 "uvarchar.pgc"
+
+
+ print_uvarchar();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 421 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 421 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 421 "uvarchar.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 422 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 422 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 422 "uvarchar.pgc"
+
+}
+
+//simple insert using uvarchar with L string inited
+void test_var_13()
+{
+/* exec sql begin declare section */
+
+
+#line 429 "uvarchar.pgc"
+ struct uvarchar_3 { int len; utext arr[ VAR_SIZE ]; } uvarchar_local_var ;
+/* exec sql end declare section */
+#line 430 "uvarchar.pgc"
+
+
+ test("test_var_13 : simple insert using uvarchar with L string inited");
+ init_var();
+
+ memset((char*)&uvarchar_local_var,0,sizeof(uvarchar_local_var));
+
+ memcpy((char*)uvarchar_local_var.arr, L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋", sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋"));
+ uvarchar_local_var.len = sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋")/4;
+
+ printf("uvarchar_local_var.len = %d\n",uvarchar_local_var.len);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 16 )",
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_3),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 442 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 442 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 442 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 444 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 444 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 444 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 16", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 447 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 447 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 447 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+
+ init_table_value();
+}
+
+//Open cursor using utext var directly in WHERE Clause
+void test_var_14()
+{
+/* exec sql begin declare section */
+
+
+#line 458 "uvarchar.pgc"
+ struct uvarchar_4 { int len; utext arr[ 20 + 1 ]; } uvarchar_local_var ;
+/* exec sql end declare section */
+#line 459 "uvarchar.pgc"
+
+ memset((char*)&uvarchar_local_var,0x00,sizeof(uvarchar_local_var));
+
+ test("test_var_14 : Open cursor using utext var directly in WHERE Clause");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 465 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 465 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 465 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_4),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 466 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 466 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 466 "uvarchar.pgc"
+
+ ECPGset_var( 0, &( uvarchar_local_var ), __LINE__);\
+ /* declare cursor_var_14 cursor for select count from tb1 where Item = $1 */
+#line 467 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_var_14 cursor for select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_4),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 468 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 468 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 468 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_var_14", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 469 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 469 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 469 "uvarchar.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_var_14", ECPGt_EOIT, ECPGt_EORT);
+#line 472 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 472 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 472 "uvarchar.pgc"
+
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 474 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 474 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 474 "uvarchar.pgc"
+
+}
+
+//Test uvarchar_var working with NULL without indicator
+void test_var_15()
+{
+/* exec sql begin declare section */
+
+
+#line 481 "uvarchar.pgc"
+ struct uvarchar_5 { int len; utext arr[ 20 + 1 ]; } uvarchar_local_var ;
+/* exec sql end declare section */
+#line 482 "uvarchar.pgc"
+
+ test("test_var_15 : Test uvarchar_var working with NULL without indicator");
+
+ memset((char*)&uvarchar_local_var,'a',sizeof(uvarchar_local_var));
+ uvarchar_local_var.len=10;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 8", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_5),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 487 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 487 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 487 "uvarchar.pgc"
+
+ print_local_uvarchar(uvarchar_local_var.arr,20);
+
+
+ memset((char*)&uvarchar_local_var,0x00,sizeof(uvarchar_local_var));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_5),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 492 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 492 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 492 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_5),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 493 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 493 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 493 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_5),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 494 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 494 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 494 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 495 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 495 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 495 "uvarchar.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//uvarchar_var working with NULL with indicator
+void test_var_16()
+{
+/* exec sql begin declare section */
+
+
+
+#line 505 "uvarchar.pgc"
+ struct uvarchar_6 { int len; utext arr[ 20 + 1 ]; } uvarchar_local_var ;
+
+#line 506 "uvarchar.pgc"
+ int uvarchar_var_ind = - 1 ;
+/* exec sql end declare section */
+#line 507 "uvarchar.pgc"
+
+ test("test_var_16 : Test uvarchar_var working with NULL with indicator");
+
+ memset((char*)&uvarchar_local_var,'a',sizeof(uvarchar_local_var));
+ uvarchar_local_var.len=0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 8", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_6),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT);
+#line 512 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 512 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 512 "uvarchar.pgc"
+
+ print_local_uvarchar(uvarchar_local_var.arr,20);
+ print_uvarchar_ind(uvarchar_var_ind);
+
+ memset((char*)&uvarchar_local_var,0x00,sizeof(uvarchar_local_var));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_6),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 517 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 517 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 517 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_6),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 518 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 518 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 518 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)20 + 1,(long)1,sizeof(struct uvarchar_6),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 519 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 519 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 519 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 520 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 520 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 520 "uvarchar.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+
+//simple select into uvarchar array
+void test_array_1()
+{
+ test("test_array_1 : simple select into uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 533 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 533 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 533 "uvarchar.pgc"
+
+
+ print_array();
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 538 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 538 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 538 "uvarchar.pgc"
+
+ print_array();
+ init_var();
+}
+
+
+//simple select using uvarchar array
+void test_array_2()
+{
+ test("test_array_2 : simple select using uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 550 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 550 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 550 "uvarchar.pgc"
+
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 553 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 553 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 553 "uvarchar.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 557 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 557 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 557 "uvarchar.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using uvarchar array
+void test_array_3()
+{
+ test("test_array_3 : simple update using uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 567 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 567 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 567 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 1",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 569 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 569 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 569 "uvarchar.pgc"
+
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 572 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 572 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 572 "uvarchar.pgc"
+
+ printf ("find %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 576 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 576 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 576 "uvarchar.pgc"
+
+ printf ("find %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using uvarchar array
+void test_array_4()
+{
+ test("test_array_4 : simple delete using uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 588 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 588 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 588 "uvarchar.pgc"
+
+
+ count = 100;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 591 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 591 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 591 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 592 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 592 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 592 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 100;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 596 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 596 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 596 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 597 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 597 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 597 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using uvarchar array
+void test_array_5()
+{
+ test("test_array_5 : simple insert using uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 609 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 609 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 609 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 11 )",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 611 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 611 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 611 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 612 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 612 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 612 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 615 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 615 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 615 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 616 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 616 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 616 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into uvarchar array
+void test_array_6()
+{
+ test("test_array_6 : prepared select into uvarchar array");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count<=?");
+#line 628 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 628 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 628 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 630 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 630 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 630 "uvarchar.pgc"
+
+
+ print_array();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 634 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 634 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 634 "uvarchar.pgc"
+
+}
+
+//prepared select using uvarchar array
+void test_array_7()
+{
+ test("test_array_7 : prepared select using uvarchar array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 642 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 642 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 642 "uvarchar.pgc"
+
+
+ count = 0;
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 645 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 645 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 645 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 646 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 646 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 646 "uvarchar.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 649 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 649 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 649 "uvarchar.pgc"
+
+}
+
+//prepared update using uvarchar array
+void test_array_8()
+{
+ test("test_array_8 : prepared update using uvarchar array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 657 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 657 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 657 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 659 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 659 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 659 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 660 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 660 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 660 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 661 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 661 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 661 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 662 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 662 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 662 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 665 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 665 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 665 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using uvarchar array
+void test_array_9()
+{
+ test("test_array_9 : prepared delete using uvarchar array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 675 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 675 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 675 "uvarchar.pgc"
+
+
+ count = 0;
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 678 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 678 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 678 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 679 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 679 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 679 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 680 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 680 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 680 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 683 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 683 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 683 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using uvarchar array
+void test_array_10()
+{
+ test("test_array_10 : prepared insert using uvarchar array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 693 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 693 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 693 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, ?)");
+#line 695 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 695 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 695 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"13",(long)2,(long)1,strlen("13"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 696 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 696 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 696 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"15",(long)2,(long)1,strlen("15"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 697 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 697 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 697 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 698 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 698 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 698 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 701 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 701 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 701 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using uvarchar array
+void test_array_11()
+{
+ test("test_array_11 : Open cursor using uvarchar array");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 712 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 712 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 712 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 713 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 713 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 713 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 715 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 715 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 715 "uvarchar.pgc"
+
+ /* declare cursor_array_11 cursor for $1 */
+#line 716 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_array_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 717 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 717 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 717 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_array_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 718 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 718 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 718 "uvarchar.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_array_11", ECPGt_EOIT, ECPGt_EORT);
+#line 720 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 720 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 720 "uvarchar.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 721 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 721 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 721 "uvarchar.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 722 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 722 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 722 "uvarchar.pgc"
+
+}
+
+//Fecth cursor into uvarchar array
+void test_array_12()
+{
+ test("test_array_12 : Fecth cursor into uvarchar array");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 731 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 731 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 731 "uvarchar.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count<=3");
+#line 732 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 732 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 732 "uvarchar.pgc"
+
+ /* declare cursor_array_12 cursor for $1 */
+#line 733 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "declare cursor_array_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 734 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 734 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 734 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 735 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 735 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 735 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_array[1]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 736 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 736 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 736 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 737 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 737 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 737 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "close cursor_array_12", ECPGt_EOIT, ECPGt_EORT);
+#line 738 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 738 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 738 "uvarchar.pgc"
+
+
+ print_array();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 742 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 742 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 742 "uvarchar.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 743 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 743 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 743 "uvarchar.pgc"
+
+}
+
+//Insert array with L string inited
+void test_array_13()
+{
+/* exec sql begin declare section */
+
+
+#line 750 "uvarchar.pgc"
+ struct uvarchar_7 { int len; utext arr[ VAR_SIZE ]; } uvarchar_array13 [ 3 ] ;
+/* exec sql end declare section */
+#line 751 "uvarchar.pgc"
+
+ test("test_array_13 : Insert array with L string inited");
+
+ memset((char*)&uvarchar_array13,0,sizeof(uvarchar_array13));
+
+ memcpy((char*)uvarchar_array13[0].arr, L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋", sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋"));
+ uvarchar_array13[0].len = sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋")/4;
+ memcpy((char*)uvarchar_array13[1].arr, L"足球篮球羽毛球乒乓球橄榄球棒球冰球", sizeof(L"足球篮球羽毛球乒乓球橄榄球棒球冰球"));
+ uvarchar_array13[1].len = sizeof(L"足球篮球羽毛球乒乓球橄榄球棒球冰球")/4;
+ memcpy((char*)uvarchar_array13[2].arr, L"世界杯每隔四年就会举行一次每次𠲖个球队", sizeof(L"世界杯每隔四年就会举行一次每次𠲖个球队"));
+ uvarchar_array13[2].len = sizeof(L"世界杯每隔四年就会举行一次每次𠲖个球队")/4;
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_uvarchar,&(uvarchar_array13[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_7),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 763 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 763 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 763 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 21 )",
+ ECPGt_uvarchar,&(uvarchar_array13[1]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_7),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 764 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 764 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 764 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 22 )",
+ ECPGt_uvarchar,&(uvarchar_array13[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_7),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 765 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 765 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 765 "uvarchar.pgc"
+
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 22 and count >= 20", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 768 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 768 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 768 "uvarchar.pgc"
+
+ print_array();
+ init_table_value();
+
+}
+
+//uvarchar array working with NULL without using indicator
+void test_array_15()
+{
+/* exec sql begin declare section */
+
+
+#line 778 "uvarchar.pgc"
+ struct uvarchar_8 { int len; utext arr[ 20 ]; } uvarchar_local_array [ 3 ] ;
+/* exec sql end declare section */
+#line 779 "uvarchar.pgc"
+
+ test("test_array_15 : Test uvarchar array working with NULL without using indicator");
+ memset(uvarchar_local_array,'a',sizeof(uvarchar_local_array));
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count >= 8 and count <= 10", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_local_array),(long)20,(long)3,sizeof(struct uvarchar_8),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 783 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 783 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 783 "uvarchar.pgc"
+
+
+ print_local_uvarchar(uvarchar_local_array[0].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[1].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[2].arr, 20);
+
+
+ memset(uvarchar_local_array,0x00,sizeof(uvarchar_local_array));
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_uvarchar,&(uvarchar_local_array[0]),(long)20,(long)1,sizeof(struct uvarchar_8),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 791 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 791 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 791 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_uvarchar,&(uvarchar_local_array[1]),(long)20,(long)1,sizeof(struct uvarchar_8),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 792 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 792 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 792 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_uvarchar,&(uvarchar_local_array[2]),(long)20,(long)1,sizeof(struct uvarchar_8),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT);
+#line 793 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 793 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 793 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 794 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 794 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 794 "uvarchar.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//uvarchar array working with NULL using indicator
+void test_array_16()
+{
+/* exec sql begin declare section */
+
+
+
+#line 804 "uvarchar.pgc"
+ struct uvarchar_9 { int len; utext arr[ 20 ]; } uvarchar_local_array [ 3 ] ;
+
+#line 805 "uvarchar.pgc"
+ int uvarchar_array_ind [ 3 ] ;
+/* exec sql end declare section */
+#line 806 "uvarchar.pgc"
+
+ test("test_array_16 : Test uvarchar array working with NULL using indicator");
+ memset(uvarchar_local_array,'a',sizeof(uvarchar_local_array));
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select Item from tb1 where count >= 8 and count <= 10", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_local_array),(long)20,(long)3,sizeof(struct uvarchar_9),
+ ECPGt_int,(uvarchar_array_ind),(long)1,(long)3,sizeof(int), ECPGt_EORT);
+#line 810 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 810 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 810 "uvarchar.pgc"
+
+
+
+ print_local_uvarchar(uvarchar_local_array[0].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[1].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[2].arr, 20);
+
+ print_uvarchar_ind(uvarchar_array_ind[0]);
+ print_uvarchar_ind(uvarchar_array_ind[1]);
+ print_uvarchar_ind(uvarchar_array_ind[2]);
+
+ memset(uvarchar_local_array,0x00,sizeof(uvarchar_local_array));
+ uvarchar_array_ind[0]=-1;
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 18 )",
+ ECPGt_uvarchar,&(uvarchar_local_array[0]),(long)20,(long)1,sizeof(struct uvarchar_9),
+ ECPGt_int,&(uvarchar_array_ind[0]),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 823 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 823 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 823 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 19 )",
+ ECPGt_uvarchar,&(uvarchar_local_array[1]),(long)20,(long)1,sizeof(struct uvarchar_9),
+ ECPGt_int,&(uvarchar_array_ind[0]),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 824 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 824 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 824 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_uvarchar,&(uvarchar_local_array[2]),(long)20,(long)1,sizeof(struct uvarchar_9),
+ ECPGt_int,&(uvarchar_array_ind[0]),(long)1,(long)1,sizeof(int), ECPGt_EOIT, ECPGt_EORT);
+#line 825 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 825 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 825 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 0, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item is null", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT);
+#line 826 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 826 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 826 "uvarchar.pgc"
+
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+void test_all()
+{
+ test_var_1();
+ test_var_2();
+ test_var_3();
+ test_var_4();
+ test_var_5();
+ test_var_6();
+ test_var_7();
+ test_var_8();
+ test_var_9();
+ test_var_10();
+ test_var_11();
+ test_var_12();
+ test_var_13();
+ test_var_14();
+ test_var_15();
+ test_var_16();
+ test_array_1();
+ test_array_2();
+ test_array_3();
+ test_array_4();
+ test_array_5();
+ test_array_6();
+ test_array_7();
+ test_array_8();
+ test_array_9();
+ test_array_10();
+ test_array_11();
+ test_array_12();
+ test_array_13();
+ test_array_15();
+ test_array_16();
+}
+
+int main()
+{
+// ECPGdebug(1, stderr);
+ if(test_init() !=0)
+ return -1;
+
+ test_all();
+ test_finish();
+
+ return 0;
+}
diff --git a/src/interfaces/ecpg/test/expected/sql-uvarchar.stderr b/src/interfaces/ecpg/test/expected/sql-uvarchar.stderr
new file mode 100644
index 0000000..158fd24
--- /dev/null
+++ b/src/interfaces/ecpg/test/expected/sql-uvarchar.stderr
@@ -0,0 +1 @@
+SQL error:
diff --git a/src/interfaces/ecpg/test/expected/sql-uvarchar.stdout b/src/interfaces/ecpg/test/expected/sql-uvarchar.stdout
new file mode 100644
index 0000000..202cd5c
--- /dev/null
+++ b/src/interfaces/ecpg/test/expected/sql-uvarchar.stdout
@@ -0,0 +1,252 @@
+
+test_var_1 : simple select into uvarchar var
+---->uvarchar variable,len=15:
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+uvarchar_var_ind = 0
+---->uvarchar variable,len=17:
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+uvarchar_var_ind = 0
+---->uvarchar variable,len=19:
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+uvarchar_var_ind = 0
+---->uvarchar variable,len=20:
+0x4E9A 0x6D32 0x6B27 0x6D32 0x975E 0x6D32 0x5927 0x6D0B
+0x6D32 0x5317 0x7F8E 0x6D32 0x5357 0x7F8E 0x6D32 0x5357
+0x6781 0x6D32 0x6CA1 0x6709
+uvarchar_var_ind = 23
+
+test_var_2 : simple select using uvarchar var
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_3 : simple update using uvarchar
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_var_4 : simple delete using uvarchar var
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_5 : simple insert using uvarchar
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_6 : prepared select into uvarchar var
+---->uvarchar variable,len=15:
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar variable,len=19:
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+
+test_var_7 : prepared select using uvarchar var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_8 : prepared update using uvarchar var
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_9 : prepared delete using uvarchar var
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_10 : prepared insert using uvarchar var
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_11 : Open cursor using uvarchar var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_12 : Fecth cursor into uvarchar var
+---->uvarchar variable,len=15:
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+
+test_var_13 : simple insert using uvarchar with L string inited
+uvarchar_local_var.len = 16
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+---->uvarchar variable,len=15:
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+uvarchar_var_ind = 0
+
+test_var_14 : Open cursor using utext var directly in WHERE Clause
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_15 : Test uvarchar_var working with NULL without indicator
+---->uvarchar variable,len=20:
+0x0000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+found 6 rows for Item being NULL
+
+test_var_16 : Test uvarchar_var working with NULL with indicator
+---->uvarchar variable,len=20:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+uvarchar_var_ind = -1
+found 6 rows for Item being NULL
+
+test_array_1 : simple select into uvarchar array
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=17)
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+---->uvarchar array[2]:(len=19)
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar array[2]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_2 : simple select using uvarchar array
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_3 : simple update using uvarchar array
+find 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+find 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_4 : simple delete using uvarchar array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_5 : simple insert using uvarchar array
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_6 : prepared select into uvarchar array
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=17)
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+---->uvarchar array[2]:(len=19)
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_7 : prepared select using uvarchar array
+count=1 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_8 : prepared update using uvarchar array
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_9 : prepared delete using uvarchar array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_10 : prepared insert using uvarchar array
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_11 : Open cursor using uvarchar array
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_12 : Fecth cursor into uvarchar array
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=17)
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+---->uvarchar array[2]:(len=19)
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_13 : Insert array with L string inited
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=17)
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+---->uvarchar array[2]:(len=19)
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_15 : Test uvarchar array working with NULL without using indicator
+---->uvarchar variable,len=20:
+0x0000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar variable,len=20:
+0x0000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar variable,len=20:
+0x0000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+found 6 rows for Item being NULL
+
+test_array_16 : Test uvarchar array working with NULL using indicator
+---->uvarchar variable,len=20:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar variable,len=20:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar variable,len=20:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+uvarchar_var_ind = -1
+uvarchar_var_ind = -1
+uvarchar_var_ind = -1
+found 6 rows for Item being NULL
diff --git a/src/interfaces/ecpg/test/sql/Makefile b/src/interfaces/ecpg/test/sql/Makefile
index b7bc034..ed50ad7 100644
--- a/src/interfaces/ecpg/test/sql/Makefile
+++ b/src/interfaces/ecpg/test/sql/Makefile
@@ -23,9 +23,17 @@ TESTS = array array.c \
quote quote.c \
show show.c \
twophase twophase.c \
- insupd insupd.c
+ insupd insupd.c \
+ utext utext.c \
+ uvarchar uvarchar.c
all: $(TESTS)
oldexec.c: oldexec.pgc $(ECPG_TEST_DEPENDENCIES)
$(ECPG) -r questionmarks -o $@ $<
+
+utext.c: utext.pgc $(ECPG_TEST_DEPENDENCIES)
+ $(ECPG) --enable-utext -r no_indicator -o $@ $<
+
+uvarchar.c: uvarchar.pgc $(ECPG_TEST_DEPENDENCIES)
+ $(ECPG) --enable-utext -r no_indicator -o $@ $<
diff --git a/src/interfaces/ecpg/test/sql/utext.pgc b/src/interfaces/ecpg/test/sql/utext.pgc
new file mode 100755
index 0000000..411de30
--- /dev/null
+++ b/src/interfaces/ecpg/test/sql/utext.pgc
@@ -0,0 +1,1477 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <wchar.h>
+EXEC SQL INCLUDE ../regression;
+
+#define test(msg) printf("\n%s\n",msg)
+#define VAR_SIZE 20
+#define ARRAY_SIZE 4
+#define P_VAR_SIZE 22
+
+
+/* Following is UTF8 and UTF16/UTF32 characters mapping table */
+/*太𠮷𠜱平洋𠱓大西洋印度洋北冰洋
+utf16
+0x592A 0xD842 0xDFB7 0xD841 0xDF31 0x5E73 0x6D0B 0xD843
+0xDC53 0x5927 0x897F 0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317
+0x51B0 0x6D0B 0x0000,0x0000
+
+utf32
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B
+*/
+
+/*足球篮球羽毛球乒乓球橄榄球棒球冰球
+utf16
+0x8DB3,0x7403,0x7BEE,0x7403,0x7FBD,0x6BDB,0x7403,0x4E52,
+0x4E53,0x7403,0x6A44,0x6984,0x7403,0x68D2,0x7403,0x51B0,
+0x7403,0x0000,0x0000,0x0000
+
+utf32
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403
+*/
+
+/*世界杯每隔四年就会举行一次每次𠲖个球队
+utf16
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0xD843
+0xDC96 0x4E2A 0x7403 0x961F
+
+utf32
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F
+*/
+
+/* 亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲
+0x4E9A,0x6D32,0x6B27,0x6D32,0x975E,0x6D32,0x5927,0x6D0B,
+0x6D32,0x5317,0x7F8E,0x6D32,0x5357,0x7F8E,0x6D32,0x5357,
+0x6781,0x6D32,0x6CA1,0x6709,0x5317,0x6781,0x6D32
+*/
+/* 太𠮷
+0x592A 0xD842 0xDFB7
+*/
+void init_var(void);
+void test_var_1(void);
+void test_var_2(void);
+void test_var_3(void);
+void test_var_4(void);
+void test_var_5(void);
+void test_var_6(void);
+void test_var_7(void);
+void test_var_8(void);
+void test_var_9(void);
+void test_var_10(void);
+void test_var_11(void);
+void test_var_12(void);
+void test_var_13(void);
+void test_var_14(void);
+void test_var_15(void);
+void test_var_16(void);
+void test_array_1(void);
+void test_array_2(void);
+void test_array_3(void);
+void test_array_4(void);
+void test_array_5(void);
+void test_array_6(void);
+void test_array_7(void);
+void test_array_8(void);
+void test_array_9(void);
+void test_array_10(void);
+void test_array_11(void);
+void test_array_12(void);
+void test_array_13(void);
+void test_array_14(void);
+void test_array_15(void);
+void test_array_16(void);
+void test_pvar_1(void);
+void test_pvar_2(void);
+void test_pvar_3(void);
+void test_pvar_4(void);
+void test_pvar_5(void);
+void test_pvar_6(void);
+void test_pvar_7(void);
+void test_pvar_8(void);
+void test_pvar_9(void);
+void test_pvar_10(void);
+void test_pvar_11(void);
+void test_pvar_12(void);
+void test_pvar_13(void);
+void test_pvar_14(void);
+void test_pvar_15(void);
+void test_pvar_16(void);
+void test_pvar_17(void);
+void test_pvar_18(void);
+void test_pvar_19(void);
+void test_pp_var_1(void);
+void test_pp_var_2(void);
+void test_pp_var_3(void);
+void test_desc_1(void);
+void test_all(void);
+
+void print_utext(utext *utext_var);
+void print_utext_str(utext *utext_var);
+void print_utext_ind(int utext_var_ind);
+void print_utext_size(utext *utext_var,int size);
+void print_array(void *array, int array_size, int var_size);
+void print_array_with_index(int index);
+int test_init(void);
+void test_finish(void);
+void init_table_value(void);
+
+EXEC SQL BEGIN DECLARE SECTION;
+ int utext_var_size = 20;
+ int utext_array_size = 4;
+ int count=0;
+ int total_tuples = 0;
+ int i;
+ char char_var[10]={0xF0,0x90,0x90,0xB7};
+
+ utext utext_var[VAR_SIZE];
+ utext utext_array[ARRAY_SIZE][VAR_SIZE];
+ utext utext_input_var[20]={0x592a,0xd842, 0xdfb7, 0x0000};
+ utext *p_utext_var = NULL;
+
+
+ int utext_var_ind;
+ int count_array[4]={1,2,3,4};
+
+EXEC SQL END DECLARE SECTION;
+
+void print_utext(utext *utext_var)
+{
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<VAR_SIZE; i++)
+ {
+ printf ("0x%08X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void print_utext_str(utext *utext_var)
+{
+ int i=0;
+ printf("======print utext_var_str content======\n");
+ if (utext_var==0)
+ {
+ printf("utext_var is NULL\n");
+ printf("\n======End utext_var_str content======\n");
+ return;
+ }
+ while(utext_var[i] != 0)
+ {
+ printf ("0x%08X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ i++;
+ if(i>100)
+ break;
+ }
+ printf("\n======End utext_var_str content======\n");
+}
+
+void print_utext_size(utext *utext_var,int size)
+{
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<size; i++)
+ {
+ printf ("0x%08X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void print_utext_ind(int utext_var_ind)
+{
+ printf("utext_var_ind = %d\n",utext_var_ind);
+}
+void print_array(void *array, int array_size, int var_size)
+{
+ int i,j;
+
+ for(i=0; i<array_size; i++)
+ {
+ utext *utext_array = &((utext*)array)[i*var_size];
+
+ printf ("---->array[%d]:\n", i);
+
+ for(j=0; j<var_size; j++)
+ {
+ printf ("0x%08X ", utext_array[j]);
+ if(j>6 && (j+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+ }
+
+ printf("\n");
+}
+
+int test_init()
+{
+ p_utext_var = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ if(!p_utext_var)
+ {
+ printf("Error: failed allco memory for p_utext_var. \n");
+ return -1;
+ }
+
+ EXEC SQL CONNECT TO REGRESSDB1;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+ EXEC SQL WHENEVER SQLWARNING SQLPRINT;
+ EXEC SQL WHENEVER SQLERROR SQLPRINT;
+
+ EXEC SQL SET client_encoding='UTF8';
+
+ //initialization of test table
+ EXEC SQL CREATE TABLE IF NOT EXISTS tb1 (Item varchar, Count integer);
+
+ init_table_value();
+
+ return 0;
+}
+
+void test_finish()
+{
+ EXEC SQL DROP TABLE tb1;
+ EXEC SQL DISCONNECT ALL;
+}
+
+void init_table_value()
+{
+ EXEC SQL TRUNCATE tb1;
+
+ EXEC SQL INSERT INTO tb1 VALUES ('太𠮷𠜱平洋𠱓大西洋印度洋北冰洋', 1);
+ EXEC SQL INSERT INTO tb1 VALUES ('足球篮球羽毛球乒乓球橄榄球棒球冰球', 2);
+ EXEC SQL INSERT INTO tb1 VALUES ('世界杯每隔四年就会举行一次每次𠲖个球队', 3);
+ EXEC SQL INSERT INTO tb1 VALUES ('亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲', 4);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(8);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(9);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(10);
+}
+
+void init_var()
+{
+ count = 0;
+ memset(utext_var,'a',sizeof(utext_var));
+ memset(utext_array,'a',sizeof(utext_array));
+ memset(p_utext_var,'a',P_VAR_SIZE*sizeof(utext));
+}
+
+//simple select into utext var
+void test_var_1()
+{
+ test("test_var_1: simple select into utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=1;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=2;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=3;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=4;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+}
+
+
+//simple select using utext var
+void test_var_2()
+{
+ test("test_var_2: simple select using utext var");
+ init_var();
+
+ count = 0;
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:utext_var;
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:utext_var;
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using utext var
+void test_var_3()
+{
+ test("test_var_3: simple update using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL UPDATE tb1 SET Item=:utext_var WHERE Count=2;
+ EXEC SQL UPDATE tb1 SET Item=:utext_var WHERE Count=3;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext var
+void test_var_4()
+{
+ test("test_var_4 : simple delete using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext var
+void test_var_5()
+{
+ test("test_var_5 : simple insert using utext");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL INSERT INTO tb1 values (:utext_var, 11);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL INSERT INTO tb1 values (:utext_var, 13);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext var
+void test_var_6()
+{
+ test("test_var_6 : prepared select into utext var");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=?";
+
+ EXEC SQL EXECUTE stmt INTO :utext_var USING 1;
+ print_utext(utext_var);
+
+ EXEC SQL EXECUTE stmt INTO :utext_var USING 3;
+ print_utext(utext_var);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared select using utext var
+void test_var_7()
+{
+ test("test_var_7 : prepared select using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :utext_var;
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using utext var
+void test_var_8()
+{
+ test("test_var_8 : prepared update using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :utext_var, 1;
+ EXEC SQL EXECUTE stmt USING :utext_var, 2;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using utext var
+void test_var_9()
+{
+ test("test_var_9 : prepared delete using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using utext var
+void test_var_10()
+{
+ test("test_var_10 : prepared insert using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, 13)";
+ EXEC SQL EXECUTE stmt USING :utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using utext var
+void test_var_11()
+{
+ test("test_var_11 : Open cursor using utext var");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_var_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_var_11 USING :utext_var;
+ EXEC SQL FETCH cursor_var_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_var_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into utext var
+void test_var_12()
+{
+ test("test_var_12 : Fecth cursor into utext var");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=1";
+ EXEC SQL DECLARE cursor_var_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_var_12;
+ EXEC SQL FETCH cursor_var_12 INTO :utext_var;
+ EXEC SQL CLOSE cursor_var_12;
+
+ print_utext(utext_var);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//simple insert utext var with L string inited
+void test_var_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[]=L"足球篮球羽毛球";
+EXEC SQL END DECLARE SECTION;
+ test("test_var_13 : simple insert utext var with L string inited");
+ init_var();
+
+ print_utext_str(utext_local_var);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var, 16);
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=16;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+ init_table_value();
+}
+
+//Open cursor using utext var directly in WHERE Clause
+void test_var_14()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[20+1];
+EXEC SQL END DECLARE SECTION;
+ memset(utext_local_var,'a',sizeof(utext_local_var));
+
+ test("test_var_14 : Open cursor using utext var directly in WHERE Clause");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_local_var FROM tb1 WHERE Count=3;
+ EXEC SQL DECLARE cursor_var_14 CURSOR FOR SELECT Count FROM tb1 WHERE Item=:utext_local_var;
+ EXEC SQL OPEN cursor_var_14;
+ EXEC SQL FETCH cursor_var_14 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ EXEC SQL CLOSE cursor_var_14;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Test utext_var working with NULL without indicator
+void test_var_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[20+1];
+EXEC SQL END DECLARE SECTION;
+ test("test_var_15 : Test utext_var working with NULL without indicator");
+
+ memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_var FROM tb1 WHERE Count=8;
+ print_utext(utext_local_var);
+
+ memset(utext_local_var,0x00,sizeof(utext_local_var));
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext_var working with NULL with indicator
+void test_var_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[20+1];
+ int utext_var_ind=-1;
+EXEC SQL END DECLARE SECTION;
+ test("test_var_16 : Test utext_var working with NULL with indicator");
+
+ memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_var:utext_var_ind FROM tb1 WHERE Count=8;
+ print_utext(utext_local_var);
+ print_utext_ind(utext_var_ind);
+
+ memset(utext_local_var,0x00,sizeof(utext_local_var));
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var:utext_var_ind, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var:utext_var_ind, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var:utext_var_ind, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+
+//simple select into utext array
+void test_array_1()
+{
+ test("test_array_1 : simple select into utext array ");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array[0] FROM tb1 WHERE Count=1;
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+ init_var();
+}
+
+//simple select using utext array
+void test_array_2()
+{
+ test("test_array_2 : simple select using array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:utext_array[0];
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:utext_array[2];
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using utext array
+void test_array_3()
+{
+ test("test_array_3 : simple update using utext array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL UPDATE tb1 SET Item=:utext_array[2] WHERE Count=1;
+
+ count = 0;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("find %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("find %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext array
+void test_array_4()
+{
+ test("test_array_4 : simple delete using utext array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ count = 100;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:utext_array[0];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 100;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:utext_array[2];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext array
+void test_array_5()
+{
+ test("test_array_5 : simple insert using utext array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL INSERT INTO tb1 values (:utext_array[0], 11);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_array[2], 13);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext array
+void test_array_6()
+{
+ test("test_array_6 : prepared select into utext array");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count<=?";
+
+ EXEC SQL EXECUTE stmt INTO :utext_array USING 4;
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared select using utext array
+void test_array_7()
+{
+ test("test_array_7 : prepared select using utext array");
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :utext_array[0];
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using utext array
+void test_array_8()
+{
+ test("test_array_8 : prepared update using utext array");
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :utext_array[0], 2;
+ EXEC SQL EXECUTE stmt USING :utext_array[0], 3;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using utext array
+void test_array_9()
+{
+ test("test_array_9 : prepared delete using utext array");
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :utext_array[0];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using utext array
+void test_array_10()
+{
+ test("test_array_10 : prepared insert using utext array");
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, ?)";
+ EXEC SQL EXECUTE stmt USING :utext_array[2], 13;
+ EXEC SQL EXECUTE stmt USING :utext_array[2], 15;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using utext array
+void test_array_11()
+{
+ test("test_array_11 : Open cursor using utext array");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_array_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_array_11 USING :utext_array[2];
+ EXEC SQL FETCH cursor_array_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_array_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into utext array
+void test_array_12()
+{
+ test("test_array_12 : Fecth cursor into utext array");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count<=3";
+ EXEC SQL DECLARE cursor_array_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_array_12;
+ EXEC SQL FETCH cursor_array_12 INTO :utext_array[0];
+ EXEC SQL FETCH cursor_array_12 INTO :utext_array[1];
+ EXEC SQL FETCH cursor_array_12 INTO :utext_array[2];
+ EXEC SQL CLOSE cursor_array_12;
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Insert array with L string inited
+void test_array_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_array13[4][VAR_SIZE]={L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋",L"足球篮球羽毛球乒乓球橄榄球棒球冰球",L"世界杯每隔四年就会举行一次"};
+EXEC SQL END DECLARE SECTION;
+
+ test("test_array_13 : Insert array with L string inited");
+ init_var();
+
+
+ EXEC SQL INSERT INTO tb1 values (:utext_array13[0], 20);
+ EXEC SQL INSERT INTO tb1 values (:utext_array13[1], 21);
+ EXEC SQL INSERT INTO tb1 values (:utext_array13[2], 22);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=22 AND Count>=20;
+ print_array(utext_array,3,VAR_SIZE);
+ init_table_value();
+}
+
+//Open cursor using utext array directly in WHERE Clause
+void test_array_14()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_array[4][20+1];
+EXEC SQL END DECLARE SECTION;
+
+ test("test_array_14 : Open cursor using utext array directly in WHERE Clause");
+ memset(utext_local_array,'a',sizeof(utext_local_array));
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_local_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL DECLARE cursor_array_14 CURSOR FOR SELECT Count FROM tb1 WHERE Item=:utext_local_array[2];
+ EXEC SQL OPEN cursor_array_14;
+ EXEC SQL FETCH cursor_array_14 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_array_14;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+
+//utext array working with NULL without using indicator
+void test_array_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_array[3][20];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_15 : Test utext array working with NULL without using indicator");
+ memset(utext_local_array,'a',sizeof(utext_local_array));
+
+ EXEC SQL SELECT Item INTO :utext_local_array FROM tb1 WHERE Count>=8 and Count<=10;
+
+ print_array((void**)utext_local_array,3,20);
+
+ memset(utext_local_array,0x00,sizeof(utext_local_array));
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext array working with NULL using indicator
+void test_array_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_array[3][20];
+ int utext_array_ind[3];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_16 : Test utext array working with NULL using indicator");
+ memset(utext_local_array,'a',sizeof(utext_local_array));
+
+ EXEC SQL SELECT Item INTO :utext_local_array:utext_array_ind FROM tb1 WHERE Count>=8 and Count<=10;
+
+ print_array((void**)utext_local_array,3,20);
+ print_utext_ind(utext_array_ind[0]);
+ print_utext_ind(utext_array_ind[1]);
+ print_utext_ind(utext_array_ind[2]);
+
+ memset(utext_local_array,0x00,sizeof(utext_local_array));
+ utext_array_ind[0]=-1;
+ utext_array_ind[1]=-1;
+ utext_array_ind[2]=-1;
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[0]:utext_array_ind[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[1]:utext_array_ind[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[2]:utext_array_ind[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+
+//simple select into utext pointer
+void test_pvar_1()
+{
+ test("test_pvar_1 : simple select into utext pointer");
+ EXEC SQL SELECT Item INTO :p_utext_var:utext_var_ind FROM tb1 WHERE Count=1;
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var:utext_var_ind FROM tb1 WHERE Count=2;
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var:utext_var_ind FROM tb1 WHERE Count=3;
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var:utext_var_ind FROM tb1 WHERE Count=4;
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ init_table_value();
+}
+
+
+//simple select using utext pointer
+void test_pvar_2()
+{
+ test("test_pvar_2 : simple select using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:p_utext_var;
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:p_utext_var;
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+
+//simple update using utext pointer
+void test_pvar_3()
+{
+ test("test_pvar_3 : simple update using utext pointer");
+ init_var();
+
+ count = 0;
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL UPDATE tb1 SET Item=:p_utext_var WHERE Count=2;
+ EXEC SQL UPDATE tb1 SET Item=:p_utext_var WHERE Count=3;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext pointer
+void test_pvar_4()
+{
+ test("test_pvar_4 : simple delete using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:p_utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:p_utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext pointer
+void test_pvar_5()
+{
+ test("test_pvar_5 : simple insert using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL INSERT INTO tb1 values (:p_utext_var, 13);
+
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext pointer
+void test_pvar_6()
+{
+ test("test_pvar_6 : prepared select into utext pointer");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=?";
+
+ EXEC SQL EXECUTE stmt INTO :p_utext_var USING 1;
+ print_utext(p_utext_var);
+
+ init_var();
+ EXEC SQL EXECUTE stmt INTO :p_utext_var USING 3;
+ print_utext(p_utext_var);
+ init_table_value();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+}
+
+//prepared select using utext pointer
+void test_pvar_7()
+{
+ test("test_pvar_7 : prepared select using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :p_utext_var;
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using utext pointer
+void test_pvar_8()
+{
+ test("test_pvar_8 : prepared update using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :p_utext_var, 1;
+ EXEC SQL EXECUTE stmt USING :p_utext_var, 2;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using utext pointer
+void test_pvar_9()
+{
+ test("test_pvar_9 : prepared delete using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :p_utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using utext pointer
+void test_pvar_10()
+{
+ test("test_pvar_10 : prepared insert using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, 13)";
+ EXEC SQL EXECUTE stmt USING :p_utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using utext pointer
+void test_pvar_11()
+{
+ test("test_pvar_11 : Open cursor using utext pointer");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_pvar_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_pvar_11 USING :p_utext_var;
+ EXEC SQL FETCH cursor_pvar_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_pvar_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into utext pointer
+void test_pvar_12()
+{
+ test("test_pvar_12 : Fecth cursor into utext pointer");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=1";
+ EXEC SQL DECLARE cursor_pvar_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_pvar_12;
+ EXEC SQL FETCH cursor_pvar_12 INTO :p_utext_var;
+ EXEC SQL CLOSE cursor_pvar_12;
+
+ print_utext(p_utext_var);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//simple insert utext pointer with L string inited
+void test_pvar_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar=L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋";
+EXEC SQL END DECLARE SECTION;
+ test("test_pvar_13 : simple insert utext pointer with L string inited");
+ init_var();
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 16);
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=16;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+ init_table_value();
+}
+
+//Open cursor using utext var directly in WHERE Clause
+void test_pvar_14()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar;
+EXEC SQL END DECLARE SECTION;
+ utext_local_pvar = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ memset(utext_local_pvar,'a',P_VAR_SIZE*sizeof(utext));
+
+ test("test_pvar_14 : Open cursor using utext var directly in WHERE Clause");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_local_pvar FROM tb1 WHERE Count=3;
+
+ EXEC SQL DECLARE cursor_pvar_14 CURSOR FOR SELECT Count FROM tb1 WHERE Item=:utext_local_pvar;
+ EXEC SQL OPEN cursor_pvar_14;
+ EXEC SQL FETCH cursor_pvar_14 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_pvar_14;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+
+//utext pointer working with NULL without using indicator
+void test_pvar_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar;
+EXEC SQL END DECLARE SECTION;
+ utext_local_pvar = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ memset(utext_local_pvar,'a',P_VAR_SIZE*sizeof(utext));
+
+ test("test_pvar_15 : Test utext pointer working with NULL without using indicator");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar FROM tb1 WHERE Count=8;
+
+ print_utext(utext_local_pvar);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext pointer working with NULL using indicator
+void test_pvar_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar;
+ int utext_pvar_ind = -1;
+EXEC SQL END DECLARE SECTION;
+ utext_local_pvar = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ memset(utext_local_pvar,'a',P_VAR_SIZE*sizeof(utext));
+
+ test("test_pvar_16 : Test utext pointer working with NULL using indicator");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar:utext_pvar_ind FROM tb1 WHERE Count=8;
+
+ print_utext(utext_local_pvar);
+ print_utext_ind(utext_pvar_ind);
+
+ memset(utext_local_pvar,0,P_VAR_SIZE*sizeof(utext));
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext uninitialized pointer getting and setting data
+void test_pvar_17()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar=0;
+ char local_str[50];
+EXEC SQL END DECLARE SECTION;
+
+ test("test_pvar_17 : Test utext uninitialized pointer getting and setting data");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar FROM tb1 WHERE Count=1;
+
+ print_utext_str(utext_local_pvar);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 18);
+ EXEC SQL SELECT Item INTO :local_str FROM tb1 WHERE Count=18;
+ printf ("Find Item = %s where Count=18\n", local_str);
+
+ init_table_value();
+}
+
+//utext uninitialized pointer working with NULL without using indicator
+void test_pvar_18()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar=0;
+EXEC SQL END DECLARE SECTION;
+
+ test("test_pvar_18 : Test utext uninitialized pointer working with NULL without using indicator");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar FROM tb1 WHERE Count=8;
+
+ print_utext_size(utext_local_pvar,1);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext uninitialized pointer working with NULL using indicator
+void test_pvar_19()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar=0;
+ int utext_pvar_ind = -1;
+EXEC SQL END DECLARE SECTION;
+
+ test("test_pvar_19 : Test utext uninitialized pointer working with NULL using indicator");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar:utext_pvar_ind FROM tb1 WHERE Count=8;
+
+ print_utext_size(utext_local_pvar,1);
+ print_utext_ind(utext_pvar_ind);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//Test select into host varaibles from descriptor
+void test_desc_1()
+{
+ EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[20];
+ utext *utext_local_pvar=0;
+ utext **utext_local_ppvar=0;
+ char desc1[8] = "outdesc";
+ EXEC SQL END DECLARE SECTION;
+
+// ECPGdebug(1, stderr);
+
+ EXEC SQL ALLOCATE DESCRIPTOR indesc;
+ EXEC SQL ALLOCATE DESCRIPTOR :desc1;
+
+ EXEC SQL SELECT Item,Count INTO SQL DESCRIPTOR :desc1 FROM tb1 WHERE Count=1;
+
+ EXEC SQL GET DESCRIPTOR :desc1 VALUE 1 :utext_local_var = DATA;
+ print_utext(utext_local_var);
+
+ EXEC SQL GET DESCRIPTOR :desc1 VALUE 1 :utext_local_pvar = DATA;
+ print_utext_str(utext_local_pvar);
+
+
+ EXEC SQL SELECT Item,Count INTO SQL DESCRIPTOR :desc1 FROM tb1 WHERE Count<=3;
+ EXEC SQL GET DESCRIPTOR :desc1 VALUE 1 :utext_local_ppvar = DATA;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_str(utext_local_ppvar[2]);
+
+
+ EXEC SQL SELECT Item,Count INTO SQL DESCRIPTOR :desc1 FROM tb1 WHERE Count<=10 and Count>=8;
+ EXEC SQL GET DESCRIPTOR :desc1 VALUE 1 :utext_local_ppvar = DATA;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_str(utext_local_ppvar[2]);
+
+
+ EXEC SQL DEALLOCATE DESCRIPTOR indesc;
+ EXEC SQL DEALLOCATE DESCRIPTOR :desc1;
+}
+
+
+//Test uninitialized pointer to pointer utext_var without initialize
+void test_pp_var_1()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext **utext_local_ppvar=0;
+ int *utext_local_ppvar_ind=0;
+ char utext_local_char[80];
+EXEC SQL END DECLARE SECTION;
+ test("test_pp_var_1 : Test pointer to pointer utext_var without initialize");
+
+ //memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_ppvar:utext_local_ppvar_ind FROM tb1 WHERE Count<=3;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_ind(utext_local_ppvar_ind[0]);
+
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_ind(utext_local_ppvar_ind[1]);
+
+ print_utext_str(utext_local_ppvar[2]);
+ print_utext_ind(utext_local_ppvar_ind[2]);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[1], 28);
+ EXEC SQL SELECT Item INTO :utext_local_char FROM tb1 WHERE Count=28;
+ printf("%s\n",utext_local_char);
+
+ init_table_value();
+}
+
+
+//Test uninitialized pointer to pointer utext_var working with NULL without indicator
+void test_pp_var_2()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext **utext_local_ppvar=0;
+EXEC SQL END DECLARE SECTION;
+ test("test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator");
+
+ //memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_ppvar FROM tb1 WHERE Count>=8 and Count<=10;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_str(utext_local_ppvar[2]);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//Test uninitialized pointer to pointer utext_var working with NULL with indicator
+void test_pp_var_3()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext **utext_local_ppvar=0;
+ int *utext_local_ppvar_ind=0;
+EXEC SQL END DECLARE SECTION;
+ test("test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator");
+
+ //memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_ppvar:utext_local_ppvar_ind FROM tb1 WHERE Count>=8 and Count<=10;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_ind(utext_local_ppvar_ind[0]);
+
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_ind(utext_local_ppvar_ind[1]);
+
+ print_utext_str(utext_local_ppvar[2]);
+ print_utext_ind(utext_local_ppvar_ind[2]);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+void test_all()
+{
+ test_var_1();
+ test_var_2();
+ test_var_3();
+ test_var_4();
+ test_var_5();
+ test_var_6();
+ test_var_7();
+ test_var_8();
+ test_var_9();
+ test_var_10();
+ test_var_11();
+ test_var_12();
+ test_var_13();
+ test_var_14();
+ test_var_15();
+ test_var_16();
+ test_array_1();
+ test_array_2();
+ test_array_3();
+ test_array_4();
+ test_array_5();
+ test_array_6();
+ test_array_7();
+ test_array_8();
+ test_array_9();
+ test_array_10();
+ test_array_11();
+ test_array_12();
+ test_array_13();
+ test_array_14();
+ test_array_15();
+ test_array_16();
+ test_pvar_1();
+ test_pvar_2();
+ test_pvar_3();
+ test_pvar_4();
+ test_pvar_5();
+ test_pvar_6();
+ test_pvar_7();
+ test_pvar_8();
+ test_pvar_9();
+ test_pvar_10();
+ test_pvar_11();
+ test_pvar_12();
+ test_pvar_13();
+ test_pvar_14();
+ test_pvar_15();
+ test_pvar_16();
+ test_pvar_17();
+ test_pvar_18();
+ test_pvar_19();
+ test_pp_var_1();
+ test_pp_var_2();
+ test_pp_var_3();
+}
+
+int main(int argc, char *argv[])
+{
+// ECPGdebug(1, stderr);
+ if(test_init() !=0)
+ return -1;
+
+ test_all();
+ test_finish();
+
+ return 0;
+}
diff --git a/src/interfaces/ecpg/test/sql/uvarchar.pgc b/src/interfaces/ecpg/test/sql/uvarchar.pgc
new file mode 100755
index 0000000..4dd894a
--- /dev/null
+++ b/src/interfaces/ecpg/test/sql/uvarchar.pgc
@@ -0,0 +1,877 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+EXEC SQL INCLUDE ../regression;
+
+#define test(msg) printf("\n%s\n",msg)
+#define VAR_SIZE 20
+#define ARRAY_SIZE 4
+
+/* Following is UTF8 and UTF16 characters mapping table */
+/*太𠮷𠜱平洋𠱓大西洋印度洋北冰洋
+0x592A,0x5E73,0x6D0B,0x5927,0x897F,0x6D0B,0x5370,0x5EA6,
+0x6D0B,0x5317,0x51B0,0x6D0B,0x548C,0x5357,0x51B0,0x6D0B,
+0x0000,0x0000,0x0000,0x0000*/
+
+/*足球篮球羽毛球乒乓球橄榄球棒球冰球
+0x8DB3,0x7403,0x7BEE,0x7403,0x7FBD,0x6BDB,0x7403,0x4E52,
+0x4E53,0x7403,0x6A44,0x6984,0x7403,0x68D2,0x7403,0x51B0,
+0x7403,0x0000,0x0000,0x0000
+*/
+
+/*世界杯每隔四年就会举行一次每次𠲖个球队
+0x4E16,0x754C,0x676F,0x6BCF,0x9694,0x56DB,0x5E74,0x5C31,
+0x4F1A,0x4E3E,0x884C,0x4E00,0x6B21,0x6BCF,0x6B21,0x0033,
+0x0032,0x4E2A,0x7403,0x961F
+*/
+
+/* 亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲
+0x4E9A,0x6D32,0x6B27,0x6D32,0x975E,0x6D32,0x5927,0x6D0B,
+0x6D32,0x5317,0x7F8E,0x6D32,0x5357,0x7F8E,0x6D32,0x5357,
+0x6781,0x6D32,0x6CA1,0x6709,0x5317,0x6781,0x6D32
+*/
+
+EXEC SQL BEGIN DECLARE SECTION;
+ UVARCHAR uvarchar_var[VAR_SIZE];
+ UVARCHAR uvarchar_array[ARRAY_SIZE][VAR_SIZE];
+
+ int uvarchar_var_ind;
+
+ int count;
+ int count_array[4]={1,2,3,4};
+ int total_tuples = 0;
+EXEC SQL END DECLARE SECTION;
+
+void print_uvarchar(void);
+void print_uvarchar_ind(int uvarchar_var_ind);
+void print_local_uvarchar(utext *utext_var, int var_len);
+void print_array(void);
+int test_init(void);
+void test_finish(void);
+void init_table_value(void);
+void init_var(void);
+
+void test_var_1(void);
+void test_var_2(void);
+void test_var_3(void);
+void test_var_4(void);
+void test_var_5(void);
+void test_var_6(void);
+void test_var_7(void);
+void test_var_8(void);
+void test_var_9(void);
+void test_var_10(void);
+void test_var_11(void);
+void test_var_12(void);
+void test_var_13(void);
+void test_var_14(void);
+void test_var_15(void);
+void test_var_16(void);
+void test_array_1(void);
+void test_array_2(void);
+void test_array_3(void);
+void test_array_4(void);
+void test_array_5(void);
+void test_array_6(void);
+void test_array_7(void);
+void test_array_8(void);
+void test_array_9(void);
+void test_array_10(void);
+void test_array_11(void);
+void test_array_12(void);
+void test_array_13(void);
+void test_array_14(void);
+void test_array_15(void);
+void test_array_16(void);
+
+void test_all(void);
+
+void print_uvarchar()
+{
+ int i;
+
+ printf ("---->uvarchar variable,len=%d:\n",uvarchar_var.len);
+ for(i=0; i<VAR_SIZE; i++)
+ {
+ printf ("0x%04X ", uvarchar_var.arr[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_uvarchar_ind(int uvarchar_var_ind)
+{
+ printf("uvarchar_var_ind = %d\n",uvarchar_var_ind);
+}
+
+void print_local_uvarchar(utext *utext_var, int var_len)
+{
+ int i;
+
+ printf ("---->uvarchar variable,len=%d:\n",var_len);
+ for(i=0; i<20; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_array()
+{
+ int i,j;
+
+ for(i=0; i<ARRAY_SIZE; i++)
+ {
+ printf ("---->uvarchar array[%d]:(len=%d)\n", i,uvarchar_array[i].len);
+
+ for(j=0; j<VAR_SIZE; j++)
+ {
+ printf ("0x%04X ", uvarchar_array[i].arr[j]);
+ if(j>6 && (j+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+ }
+
+ printf("\n");
+}
+
+int test_init()
+{
+ EXEC SQL CONNECT TO REGRESSDB1;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+ EXEC SQL WHENEVER SQLWARNING SQLPRINT;
+ EXEC SQL WHENEVER SQLERROR SQLPRINT;
+
+ EXEC SQL SET client_encoding='UTF8';
+
+ //initialization of test table
+ EXEC SQL CREATE TABLE tb1 (Item varchar, Count integer);
+
+ init_table_value();
+
+ return 0;
+}
+
+void test_finish()
+{
+ EXEC SQL DROP TABLE tb1;
+ EXEC SQL DISCONNECT ALL;
+}
+
+
+void init_table_value()
+{
+ EXEC SQL TRUNCATE tb1;
+
+ EXEC SQL INSERT INTO tb1 VALUES ('太𠮷𠜱平洋𠱓大西洋印度洋北冰洋', 1);
+ EXEC SQL INSERT INTO tb1 VALUES ('足球篮球羽毛球乒乓球橄榄球棒球冰球', 2);
+ EXEC SQL INSERT INTO tb1 VALUES ('世界杯每隔四年就会举行一次每次𠲖个球队', 3);
+ EXEC SQL INSERT INTO tb1 VALUES ('亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲', 4);
+
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(8);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(9);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(10);
+}
+
+void init_var()
+{
+ int i;
+ memset((void*)&uvarchar_var,'a',sizeof(uvarchar_var));
+ uvarchar_var.len = 0;
+
+ memset((char*)uvarchar_array,'a',sizeof(uvarchar_array)*ARRAY_SIZE);
+
+ for(i=0;i<ARRAY_SIZE;i++)
+ {
+ uvarchar_array[i].len = 0;
+ }
+
+ uvarchar_var_ind = 0;
+}
+
+//simple select into uvarchar
+void test_var_1()
+{
+ test("test_var_1 : simple select into uvarchar var");
+ init_var();
+ //print_uvarchar();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=1;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=2;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=3;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=4;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+}
+
+
+//simple select using uvarchar
+void test_var_2()
+{
+ test("test_var_2 : simple select using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=1;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:uvarchar_var;
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:uvarchar_var;
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using uvarchar
+void test_var_3()
+{
+ test("test_var_3 : simple update using uvarchar");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=1;
+ EXEC SQL UPDATE tb1 SET Item=:uvarchar_var WHERE Count=2;
+ EXEC SQL UPDATE tb1 SET Item=:uvarchar_var WHERE Count=3;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using uvarchar var
+void test_var_4()
+{
+ test("test_var_4 : simple delete using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=1;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:uvarchar_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:uvarchar_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using uvarchar var
+void test_var_5()
+{
+ test("test_var_5 : simple insert using uvarchar");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=1;
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_var, 11);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_var, 13);
+
+
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into uvarchar var
+void test_var_6()
+{
+ test("test_var_6 : prepared select into uvarchar var");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=?";
+
+ EXEC SQL EXECUTE stmt INTO :uvarchar_var USING 1;
+ print_uvarchar();
+
+ EXEC SQL EXECUTE stmt INTO :uvarchar_var USING 3;
+ print_uvarchar();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared select using uvarchar var
+void test_var_7()
+{
+ test("test_var_7 : prepared select using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :uvarchar_var;
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using uvarchar var
+void test_var_8()
+{
+ test("test_var_8 : prepared update using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :uvarchar_var, 1;
+ EXEC SQL EXECUTE stmt USING :uvarchar_var, 2;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using uvarchar var
+void test_var_9()
+{
+ test("test_var_9 : prepared delete using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :uvarchar_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using uvarchar var
+void test_var_10()
+{
+ test("test_var_10 : prepared insert using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, 13)";
+ EXEC SQL EXECUTE stmt USING :uvarchar_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using uvarchar var
+void test_var_11()
+{
+ test("test_var_11 : Open cursor using uvarchar var");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_var_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_var_11 USING :uvarchar_var;
+ EXEC SQL FETCH cursor_var_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_var_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into uvarchar var
+void test_var_12()
+{
+ test("test_var_12 : Fecth cursor into uvarchar var");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=1";
+ EXEC SQL DECLARE cursor_var_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_var_12;
+ EXEC SQL FETCH cursor_var_12 INTO :uvarchar_var;
+ EXEC SQL CLOSE cursor_var_12;
+
+ print_uvarchar();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//simple insert using uvarchar with L string inited
+void test_var_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_var[VAR_SIZE];
+EXEC SQL END DECLARE SECTION;
+
+ test("test_var_13 : simple insert using uvarchar with L string inited");
+ init_var();
+
+ memset((char*)&uvarchar_local_var,0,sizeof(uvarchar_local_var));
+
+ memcpy((char*)uvarchar_local_var.arr, L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋", sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋"));
+ uvarchar_local_var.len = sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋")/4;
+
+ printf("uvarchar_local_var.len = %d\n",uvarchar_local_var.len);
+
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var, 16);
+
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=16;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+
+ init_table_value();
+}
+
+//Open cursor using utext var directly in WHERE Clause
+void test_var_14()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_var[20+1];
+EXEC SQL END DECLARE SECTION;
+ memset((char*)&uvarchar_local_var,0x00,sizeof(uvarchar_local_var));
+
+ test("test_var_14 : Open cursor using utext var directly in WHERE Clause");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :uvarchar_local_var FROM tb1 WHERE Count=3;
+ EXEC SQL DECLARE cursor_var_14 CURSOR FOR SELECT Count FROM tb1 WHERE Item=:uvarchar_local_var;
+ EXEC SQL OPEN cursor_var_14;
+ EXEC SQL FETCH cursor_var_14 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ EXEC SQL CLOSE cursor_var_14;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Test uvarchar_var working with NULL without indicator
+void test_var_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_var[20+1];
+EXEC SQL END DECLARE SECTION;
+ test("test_var_15 : Test uvarchar_var working with NULL without indicator");
+
+ memset((char*)&uvarchar_local_var,'a',sizeof(uvarchar_local_var));
+ uvarchar_local_var.len=10;
+ EXEC SQL SELECT Item INTO :uvarchar_local_var FROM tb1 WHERE Count=8;
+ print_local_uvarchar(uvarchar_local_var.arr,20);
+
+
+ memset((char*)&uvarchar_local_var,0x00,sizeof(uvarchar_local_var));
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var, 18);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var, 19);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//uvarchar_var working with NULL with indicator
+void test_var_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_var[20+1];
+ int uvarchar_var_ind=-1;
+EXEC SQL END DECLARE SECTION;
+ test("test_var_16 : Test uvarchar_var working with NULL with indicator");
+
+ memset((char*)&uvarchar_local_var,'a',sizeof(uvarchar_local_var));
+ uvarchar_local_var.len=0;
+ EXEC SQL SELECT Item INTO :uvarchar_local_var:uvarchar_var_ind FROM tb1 WHERE Count=8;
+ print_local_uvarchar(uvarchar_local_var.arr,20);
+ print_uvarchar_ind(uvarchar_var_ind);
+
+ memset((char*)&uvarchar_local_var,0x00,sizeof(uvarchar_local_var));
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var:uvarchar_var_ind, 18);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var:uvarchar_var_ind, 19);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var:uvarchar_var_ind, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+
+//simple select into uvarchar array
+void test_array_1()
+{
+ test("test_array_1 : simple select into uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ print_array();
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array[0] FROM tb1 WHERE Count=1;
+ print_array();
+ init_var();
+}
+
+
+//simple select using uvarchar array
+void test_array_2()
+{
+ test("test_array_2 : simple select using uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:uvarchar_array[0];
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:uvarchar_array[2];
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using uvarchar array
+void test_array_3()
+{
+ test("test_array_3 : simple update using uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL UPDATE tb1 SET Item=:uvarchar_array[2] WHERE Count=1;
+
+ count = 0;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("find %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("find %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using uvarchar array
+void test_array_4()
+{
+ test("test_array_4 : simple delete using uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ count = 100;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:uvarchar_array[0];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 100;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:uvarchar_array[2];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using uvarchar array
+void test_array_5()
+{
+ test("test_array_5 : simple insert using uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array[0], 11);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array[2], 13);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into uvarchar array
+void test_array_6()
+{
+ test("test_array_6 : prepared select into uvarchar array");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count<=?";
+
+ EXEC SQL EXECUTE stmt INTO :uvarchar_array USING 3;
+
+ print_array();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared select using uvarchar array
+void test_array_7()
+{
+ test("test_array_7 : prepared select using uvarchar array");
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :uvarchar_array[0];
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using uvarchar array
+void test_array_8()
+{
+ test("test_array_8 : prepared update using uvarchar array");
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[0], 1;
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[0], 2;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using uvarchar array
+void test_array_9()
+{
+ test("test_array_9 : prepared delete using uvarchar array");
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[0];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using uvarchar array
+void test_array_10()
+{
+ test("test_array_10 : prepared insert using uvarchar array");
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, ?)";
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[2], 13;
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[2], 15;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using uvarchar array
+void test_array_11()
+{
+ test("test_array_11 : Open cursor using uvarchar array");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_array_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_array_11 USING :uvarchar_array[2];
+ EXEC SQL FETCH cursor_array_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_array_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into uvarchar array
+void test_array_12()
+{
+ test("test_array_12 : Fecth cursor into uvarchar array");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count<=3";
+ EXEC SQL DECLARE cursor_array_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_array_12;
+ EXEC SQL FETCH cursor_array_12 INTO :uvarchar_array[0];
+ EXEC SQL FETCH cursor_array_12 INTO :uvarchar_array[1];
+ EXEC SQL FETCH cursor_array_12 INTO :uvarchar_array[2];
+ EXEC SQL CLOSE cursor_array_12;
+
+ print_array();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Insert array with L string inited
+void test_array_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_array13[3][VAR_SIZE];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_13 : Insert array with L string inited");
+
+ memset((char*)&uvarchar_array13,0,sizeof(uvarchar_array13));
+
+ memcpy((char*)uvarchar_array13[0].arr, L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋", sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋"));
+ uvarchar_array13[0].len = sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋")/4;
+ memcpy((char*)uvarchar_array13[1].arr, L"足球篮球羽毛球乒乓球橄榄球棒球冰球", sizeof(L"足球篮球羽毛球乒乓球橄榄球棒球冰球"));
+ uvarchar_array13[1].len = sizeof(L"足球篮球羽毛球乒乓球橄榄球棒球冰球")/4;
+ memcpy((char*)uvarchar_array13[2].arr, L"世界杯每隔四年就会举行一次每次𠲖个球队", sizeof(L"世界杯每隔四年就会举行一次每次𠲖个球队"));
+ uvarchar_array13[2].len = sizeof(L"世界杯每隔四年就会举行一次每次𠲖个球队")/4;
+
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array13[0], 20);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array13[1], 21);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array13[2], 22);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=22 AND Count>=20;
+ print_array();
+ init_table_value();
+
+}
+
+//uvarchar array working with NULL without using indicator
+void test_array_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_array[3][20];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_15 : Test uvarchar array working with NULL without using indicator");
+ memset(uvarchar_local_array,'a',sizeof(uvarchar_local_array));
+
+ EXEC SQL SELECT Item INTO :uvarchar_local_array FROM tb1 WHERE Count>=8 and Count<=10;
+
+ print_local_uvarchar(uvarchar_local_array[0].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[1].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[2].arr, 20);
+
+
+ memset(uvarchar_local_array,0x00,sizeof(uvarchar_local_array));
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//uvarchar array working with NULL using indicator
+void test_array_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_array[3][20];
+ int uvarchar_array_ind[3];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_16 : Test uvarchar array working with NULL using indicator");
+ memset(uvarchar_local_array,'a',sizeof(uvarchar_local_array));
+
+ EXEC SQL SELECT Item INTO :uvarchar_local_array:uvarchar_array_ind FROM tb1 WHERE Count>=8 and Count<=10;
+
+
+ print_local_uvarchar(uvarchar_local_array[0].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[1].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[2].arr, 20);
+
+ print_uvarchar_ind(uvarchar_array_ind[0]);
+ print_uvarchar_ind(uvarchar_array_ind[1]);
+ print_uvarchar_ind(uvarchar_array_ind[2]);
+
+ memset(uvarchar_local_array,0x00,sizeof(uvarchar_local_array));
+ uvarchar_array_ind[0]=-1;
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[0]:uvarchar_array_ind[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[1]:uvarchar_array_ind[0], 19);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[2]:uvarchar_array_ind[0], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+void test_all()
+{
+ test_var_1();
+ test_var_2();
+ test_var_3();
+ test_var_4();
+ test_var_5();
+ test_var_6();
+ test_var_7();
+ test_var_8();
+ test_var_9();
+ test_var_10();
+ test_var_11();
+ test_var_12();
+ test_var_13();
+ test_var_14();
+ test_var_15();
+ test_var_16();
+ test_array_1();
+ test_array_2();
+ test_array_3();
+ test_array_4();
+ test_array_5();
+ test_array_6();
+ test_array_7();
+ test_array_8();
+ test_array_9();
+ test_array_10();
+ test_array_11();
+ test_array_12();
+ test_array_13();
+ test_array_15();
+ test_array_16();
+}
+
+int main()
+{
+// ECPGdebug(1, stderr);
+ if(test_init() !=0)
+ return -1;
+
+ test_all();
+ test_finish();
+
+ return 0;
+}
diff --git a/src/interfaces/ecpg/test/unicode/expected/utext.c b/src/interfaces/ecpg/test/unicode/expected/utext.c
new file mode 100755
index 0000000..f371a20
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/expected/utext.c
@@ -0,0 +1,3115 @@
+/* Processed by ecpg (regression mode) */
+/* These include files are added by the preprocessor */
+#define ECPG_ENABLE_UTEXT 1
+#include <ecpglib.h>
+#include <ecpgerrno.h>
+#include <sqlca.h>
+/* End of automatic include section */
+#define ECPGdebug(X,Y) ECPGdebug((X)+100,(Y))
+
+#line 1 "utext.pgc"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <tchar.h>
+
+#line 1 "./../regression.h"
+
+
+
+
+
+
+#line 5 "utext.pgc"
+
+
+#define test(msg) printf("\n%s\n",msg)
+#define VAR_SIZE 20
+#define ARRAY_SIZE 4
+#define P_VAR_SIZE 22
+
+
+/* Following is UTF8 and UTF16 characters mapping table */
+/*太𠮷𠜱平洋𠱓大西洋印度洋北冰洋
+0x592A 0xD842 0xDFB7 0xD841 0xDF31 0x5E73 0x6D0B 0xD843
+0xDC53 0x5927 0x897F 0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317
+0x51B0 0x6D0B 0x0000,0x0000*/
+
+/*足球篮球羽毛球乒乓球橄榄球棒球冰球
+0x8DB3,0x7403,0x7BEE,0x7403,0x7FBD,0x6BDB,0x7403,0x4E52,
+0x4E53,0x7403,0x6A44,0x6984,0x7403,0x68D2,0x7403,0x51B0,
+0x7403,0x0000,0x0000,0x0000
+*/
+
+/*世界杯每隔四年就会举行一次每次𠲖个球队
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0xD843
+0xDC96 0x4E2A 0x7403 0x961F
+*/
+
+/* 亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲
+0x4E9A,0x6D32,0x6B27,0x6D32,0x975E,0x6D32,0x5927,0x6D0B,
+0x6D32,0x5317,0x7F8E,0x6D32,0x5357,0x7F8E,0x6D32,0x5357,
+0x6781,0x6D32,0x6CA1,0x6709,0x5317,0x6781,0x6D32
+*/
+/* 太𠮷
+0x592A 0xD842 0xDFB7
+*/
+void init_var(void);
+void test_var_1(void);
+void test_var_2(void);
+void test_var_3(void);
+void test_var_4(void);
+void test_var_5(void);
+void test_var_6(void);
+void test_var_7(void);
+void test_var_8(void);
+void test_var_9(void);
+void test_var_10(void);
+void test_var_11(void);
+void test_var_12(void);
+void test_var_13(void);
+void test_array_1(void);
+void test_array_2(void);
+void test_array_3(void);
+void test_array_4(void);
+void test_array_5(void);
+void test_array_6(void);
+void test_array_7(void);
+void test_array_8(void);
+void test_array_9(void);
+void test_array_10(void);
+void test_array_11(void);
+void test_array_12(void);
+void test_array_13(void);
+void test_pvar_1(void);
+void test_pvar_2(void);
+void test_pvar_3(void);
+void test_pvar_4(void);
+void test_pvar_5(void);
+void test_pvar_6(void);
+void test_pvar_7(void);
+void test_pvar_8(void);
+void test_pvar_9(void);
+void test_pvar_10(void);
+void test_pvar_11(void);
+void test_pvar_12(void);
+void test_pvar_13(void);
+void test_buckinsert_1(void);
+
+void test_all(void);
+
+void print_utext(utext *utext_var);
+void print_utext_ind(int utext_var_ind);
+void print_utext_size(utext *utext_var,int size);
+void print_array(void);
+void print_array_with_index(int index);
+int test_init(void);
+void test_finish(void);
+void init_table_value(void);
+
+/* exec sql begin declare section */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+#line 93 "utext.pgc"
+ int utext_var_size = 20 ;
+
+#line 94 "utext.pgc"
+ int utext_array_size = 4 ;
+
+#line 95 "utext.pgc"
+ int count = 0 ;
+
+#line 96 "utext.pgc"
+ int total_tuples = 0 ;
+
+#line 97 "utext.pgc"
+ int i ;
+
+#line 98 "utext.pgc"
+ char char_var [ 10 ] = { 0xF0 , 0x90 , 0x90 , 0xB7 } ;
+
+#line 100 "utext.pgc"
+ utext utext_var [ VAR_SIZE ] ;
+
+#line 101 "utext.pgc"
+ utext utext_array [ ARRAY_SIZE ] [ VAR_SIZE ] ;
+
+#line 102 "utext.pgc"
+ utext utext_input_var [ 20 ] = { 0x592a , 0xd842 , 0xdfb7 , 0x0000 } ;
+
+#line 103 "utext.pgc"
+ utext utext_input_var2 [] = L"太𠮷" ;
+
+#line 104 "utext.pgc"
+ utext * p_utext_var = NULL ;
+
+#line 107 "utext.pgc"
+ int utext_var_ind ;
+
+#line 108 "utext.pgc"
+ int count_array [ 4 ] = { 1 , 2 , 3 , 4 } ;
+/* exec sql end declare section */
+#line 110 "utext.pgc"
+
+
+void print_utext(utext *utext_var)
+{
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<VAR_SIZE; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void print_utext_size(utext *utext_var,int size)
+{
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<size; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void print_utext_ind(int utext_var_ind)
+{
+ printf("utext_var_ind = %d\n",utext_var_ind);
+}
+
+void print_array()
+{
+ int i,j;
+
+ for(i=0; i<ARRAY_SIZE; i++)
+ {
+ printf ("---->array[%d]:\n", i);
+
+ for(j=0; j<VAR_SIZE; j++)
+ {
+ printf ("0x%04X ", utext_array[i][j]);
+ if(j>6 && (j+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+ }
+
+ printf("\n");
+}
+
+int test_init()
+{
+ p_utext_var = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ if(!p_utext_var)
+ {
+ printf("Error: failed allco memory for p_utext_var. \n");
+ return -1;
+ }
+
+
+ { ECPGconnect(__LINE__, 0, "regress1" , NULL, NULL , NULL, 0); }
+#line 173 "utext.pgc"
+
+
+ { ECPGsetcommit(__LINE__, "on", NULL);}
+#line 175 "utext.pgc"
+
+ /* exec sql whenever sql_warning sqlprint ; */
+#line 176 "utext.pgc"
+
+ /* exec sql whenever sqlerror sqlprint ; */
+#line 177 "utext.pgc"
+
+
+
+ //initialization of test table
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "create table if not exists tb1 ( Item varchar , count integer )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 181 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 181 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 181 "utext.pgc"
+
+
+ init_table_value();
+
+ return 0;
+}
+
+void test_finish()
+{
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "drop table tb1", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 190 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 190 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 190 "utext.pgc"
+
+ { ECPGdisconnect(__LINE__, "ALL");
+#line 191 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 191 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 191 "utext.pgc"
+
+}
+
+void init_table_value()
+{
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "truncate tb1", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 196 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 196 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 196 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋' , 1 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 198 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 198 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 198 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '足球篮球羽毛球乒乓球橄榄球棒球冰球' , 2 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 199 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 199 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 199 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '世界杯每隔四年就会举行一次每次𠲖个球队' , 3 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 200 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 200 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 200 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲' , 4 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 201 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 201 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 201 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 5 )",
+ ECPGt_char,(char_var),(long)10,(long)1,(10)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 202 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 202 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 202 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '𠮷' , 6 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 203 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 203 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 203 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '太𠮷' , 7 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 204 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 204 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 204 "utext.pgc"
+
+}
+
+void init_var()
+{
+ count = 0;
+ memset(utext_var,'a',sizeof(utext_var));
+ memset(utext_array,'a',sizeof(utext_array));
+ memset(p_utext_var,'a',P_VAR_SIZE*sizeof(utext));
+}
+
+//simple select into utext var
+void test_var_1()
+{
+ test("test_var_1: simple select into utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 221 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 221 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 221 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 2", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 226 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 226 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 226 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 231 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 231 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 231 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 4", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 236 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 236 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 236 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+}
+
+
+//simple select using utext var
+void test_var_2()
+{
+ test("test_var_2: simple select using utext var");
+ init_var();
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 250 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 250 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 250 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 251 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 251 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 251 "utext.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 255 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 255 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 255 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 256 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 256 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 256 "utext.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using utext var
+void test_var_3()
+{
+ test("test_var_3: simple update using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 266 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 266 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 266 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 2",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 267 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 267 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 267 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 3",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 268 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 268 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 268 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 269 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 269 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 269 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext var
+void test_var_4()
+{
+ test("test_var_4 : simple delete using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 281 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 281 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 281 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 282 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 282 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 282 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 283 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 283 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 283 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 286 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 286 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 286 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 287 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 287 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 287 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 288 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 288 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 288 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext var
+void test_var_5()
+{
+ test("test_var_5 : simple insert using utext");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 300 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 300 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 300 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 11 )",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 301 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 301 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 301 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 302 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 302 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 302 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 306 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 306 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 306 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 307 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 307 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 307 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 308 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 308 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 308 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext var
+void test_var_6()
+{
+ test("test_var_6 : prepared select into utext var");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=?");
+#line 320 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 320 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 320 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 322 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 322 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 322 "utext.pgc"
+
+ print_utext(utext_var);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 325 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 325 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 325 "utext.pgc"
+
+ print_utext(utext_var);
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 328 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 328 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 328 "utext.pgc"
+
+}
+
+//prepared select using utext var
+void test_var_7()
+{
+ test("test_var_7 : prepared select using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 337 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 337 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 337 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 339 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 339 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 339 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 340 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 340 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 340 "utext.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 343 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 343 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 343 "utext.pgc"
+
+}
+
+//prepared update using utext var
+void test_var_8()
+{
+ test("test_var_8 : prepared update using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 352 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 352 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 352 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 354 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 354 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 354 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 355 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 355 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 355 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 356 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 356 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 356 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 357 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 357 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 357 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 360 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 360 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 360 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using utext var
+void test_var_9()
+{
+ test("test_var_9 : prepared delete using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 371 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 371 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 371 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 373 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 373 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 373 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 374 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 374 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 374 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 375 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 375 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 375 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 378 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 378 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 378 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using utext var
+void test_var_10()
+{
+ test("test_var_10 : prepared insert using utext var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 389 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 389 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 389 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, 13)");
+#line 391 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 391 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 391 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 392 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 392 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 392 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 393 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 393 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 393 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 396 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 396 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 396 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using utext var
+void test_var_11()
+{
+ test("test_var_11 : Open cursor using utext var");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 407 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 407 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 407 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 408 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 408 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 408 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 410 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 410 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 410 "utext.pgc"
+
+ /* declare cursor_var_11 cursor for $1 */
+#line 411 "utext.pgc"
+
+ { ECPGopen("cursor_var_11", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_var_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 412 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 412 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 412 "utext.pgc"
+
+ { ECPGfetch("cursor_var_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_var_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 413 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 413 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 413 "utext.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGclose("cursor_var_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_var_11", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 415 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 415 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 415 "utext.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 416 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 416 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 416 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 417 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 417 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 417 "utext.pgc"
+
+}
+
+//Fecth cursor into utext var
+void test_var_12()
+{
+ test("test_var_12 : Fecth cursor into utext var");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 426 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 426 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 426 "utext.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=1");
+#line 427 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 427 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 427 "utext.pgc"
+
+ /* declare cursor_var_12 cursor for $1 */
+#line 428 "utext.pgc"
+
+ { ECPGopen("cursor_var_12", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_var_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 429 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 429 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 429 "utext.pgc"
+
+ { ECPGfetch("cursor_var_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_var_12", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 430 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 430 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 430 "utext.pgc"
+
+ { ECPGclose("cursor_var_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_var_12", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 431 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 431 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 431 "utext.pgc"
+
+
+ print_utext(utext_var);
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 435 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 435 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 435 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 436 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 436 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 436 "utext.pgc"
+
+}
+
+//simple insert utext var with L string inited
+void test_var_13()
+{
+/* exec sql begin declare section */
+
+
+#line 443 "utext.pgc"
+ utext utext_local_var [] = _T ( "太𠮷" ) ;
+/* exec sql end declare section */
+#line 444 "utext.pgc"
+
+ test("test_var_13 : simple insert utext var with L string inited");
+ init_var();
+
+ printf("---utext_local_var---\n");
+
+ for(i=0;i<sizeof(utext_local_var)/2;i++)
+ printf("0x%04X ",utext_local_var[i]);
+
+ printf("\n---end: utext_local_var---\n");
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 16 )",
+ ECPGt_utext,(utext_local_var),(long)sizeof(_T ( "太𠮷" ))/2,(long)1,(sizeof(_T ( "太𠮷" ))/2)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 455 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 455 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 455 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 16", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 457 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 457 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 457 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+ init_table_value();
+}
+
+//simple select into utext array
+void test_array_1()
+{
+ test("test_array_1 : simple select into utext array ");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 4", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 470 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 470 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 470 "utext.pgc"
+
+
+ print_array();
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 475 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 475 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 475 "utext.pgc"
+
+ print_array();
+ init_var();
+}
+
+//simple select using utext array
+void test_array_2()
+{
+ test("test_array_2 : simple select using array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 486 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 486 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 486 "utext.pgc"
+
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 489 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 489 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 489 "utext.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 493 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 493 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 493 "utext.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using utext array
+void test_array_3()
+{
+ test("test_array_3 : simple update using utext array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 503 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 503 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 503 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 1",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 505 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 505 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 505 "utext.pgc"
+
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 508 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 508 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 508 "utext.pgc"
+
+ printf ("find %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 512 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 512 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 512 "utext.pgc"
+
+ printf ("find %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext array
+void test_array_4()
+{
+ test("test_array_4 : simple delete using utext array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 524 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 524 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 524 "utext.pgc"
+
+
+ count = 100;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 527 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 527 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 527 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 528 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 528 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 528 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 100;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 532 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 532 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 532 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 533 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 533 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 533 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext array
+void test_array_5()
+{
+ test("test_array_5 : simple insert using utext array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 545 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 545 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 545 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 11 )",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 547 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 547 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 547 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 548 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 548 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 548 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 551 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 551 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 551 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 552 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 552 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 552 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext array
+void test_array_6()
+{
+ test("test_array_6 : prepared select into utext array");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count<=?");
+#line 564 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 564 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 564 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"4",(long)1,(long)1,strlen("4"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 566 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 566 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 566 "utext.pgc"
+
+
+ print_array();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 570 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 570 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 570 "utext.pgc"
+
+}
+
+//prepared select using utext array
+void test_array_7()
+{
+ test("test_array_7 : prepared select using utext array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 578 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 578 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 578 "utext.pgc"
+
+
+ count = 0;
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 581 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 581 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 581 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 582 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 582 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 582 "utext.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 585 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 585 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 585 "utext.pgc"
+
+}
+
+//prepared update using utext array
+void test_array_8()
+{
+ test("test_array_8 : prepared update using utext array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 593 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 593 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 593 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 595 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 595 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 595 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 596 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 596 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 596 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 597 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 597 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 597 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 598 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 598 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 598 "utext.pgc"
+
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 601 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 601 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 601 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using utext array
+void test_array_9()
+{
+ test("test_array_9 : prepared delete using utext array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 611 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 611 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 611 "utext.pgc"
+
+
+ count = 0;
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 614 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 614 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 614 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 615 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 615 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 615 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 616 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 616 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 616 "utext.pgc"
+
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 619 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 619 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 619 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using utext array
+void test_array_10()
+{
+ test("test_array_10 : prepared insert using utext array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 629 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 629 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 629 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, ?)");
+#line 631 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 631 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 631 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"13",(long)2,(long)1,strlen("13"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 632 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 632 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 632 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"15",(long)2,(long)1,strlen("15"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 633 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 633 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 633 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 634 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 634 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 634 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 637 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 637 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 637 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using utext array
+void test_array_11()
+{
+ test("test_array_11 : Open cursor using utext array");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 648 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 648 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 648 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 649 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 649 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 649 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 651 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 651 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 651 "utext.pgc"
+
+ /* declare cursor_array_11 cursor for $1 */
+#line 652 "utext.pgc"
+
+ { ECPGopen("cursor_array_11", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_array_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 653 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 653 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 653 "utext.pgc"
+
+ { ECPGfetch("cursor_array_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_array_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 654 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 654 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 654 "utext.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGclose("cursor_array_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_array_11", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 656 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 656 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 656 "utext.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 657 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 657 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 657 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 658 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 658 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 658 "utext.pgc"
+
+}
+
+//Fecth cursor into utext array
+void test_array_12()
+{
+ test("test_array_12 : Fecth cursor into utext array");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 667 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 667 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 667 "utext.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count<=3");
+#line 668 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 668 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 668 "utext.pgc"
+
+ /* declare cursor_array_12 cursor for $1 */
+#line 669 "utext.pgc"
+
+ { ECPGopen("cursor_array_12", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_array_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 670 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 670 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 670 "utext.pgc"
+
+ { ECPGfetch("cursor_array_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 671 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 671 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 671 "utext.pgc"
+
+ { ECPGfetch("cursor_array_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_utext,(utext_array[1]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 672 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 672 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 672 "utext.pgc"
+
+ { ECPGfetch("cursor_array_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 673 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 673 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 673 "utext.pgc"
+
+ { ECPGclose("cursor_array_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_array_12", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 674 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 674 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 674 "utext.pgc"
+
+
+ print_array();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 678 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 678 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 678 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 679 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 679 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 679 "utext.pgc"
+
+}
+
+//Insert array with L string inited
+void test_array_13()
+{
+ test("test_array_13 : Insert array with L string inited");
+ init_var();
+
+/* exec sql begin declare section */
+
+
+#line 689 "utext.pgc"
+ utext utext_array13 [] [ VAR_SIZE ] = { L"太𠮷" , L"𠮷太" , L"太平" } ;
+/* exec sql end declare section */
+#line 690 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_utext,(utext_array13[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 691 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 691 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 691 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 21 )",
+ ECPGt_utext,(utext_array13[1]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 692 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 692 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 692 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 22 )",
+ ECPGt_utext,(utext_array13[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 693 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 693 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 693 "utext.pgc"
+
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 22 and count >= 20", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 696 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 696 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 696 "utext.pgc"
+
+ print_array();
+ init_table_value();
+}
+
+//simple select into utext pointer
+void test_pvar_1()
+{
+ test("test_pvar_1 : simple select into utext pointer");
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 705 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 705 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 705 "utext.pgc"
+
+ print_utext(p_utext_var);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 2", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 709 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 709 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 709 "utext.pgc"
+
+ print_utext(p_utext_var);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 713 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 713 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 713 "utext.pgc"
+
+ print_utext(p_utext_var);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 4", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 717 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 717 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 717 "utext.pgc"
+
+ print_utext(p_utext_var);
+ init_var();
+ init_table_value();
+}
+
+
+//simple select using utext pointer
+void test_pvar_2()
+{
+ test("test_pvar_2 : simple select using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 730 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 730 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 730 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 731 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 731 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 731 "utext.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 735 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 735 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 735 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 736 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 736 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 736 "utext.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+
+//simple update using utext pointer
+void test_pvar_3()
+{
+ test("test_pvar_3 : simple update using utext pointer");
+ init_var();
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 748 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 748 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 748 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 2",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 749 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 749 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 749 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 3",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 750 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 750 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 750 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 751 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 751 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 751 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext pointer
+void test_pvar_4()
+{
+ test("test_pvar_4 : simple delete using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 763 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 763 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 763 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 764 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 764 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 764 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 765 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 765 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 765 "utext.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 769 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 769 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 769 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 770 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 770 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 770 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 771 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 771 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 771 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext pointer
+void test_pvar_5()
+{
+ test("test_pvar_5 : simple insert using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 783 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 783 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 783 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 784 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 784 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 784 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 786 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 786 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 786 "utext.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext pointer
+void test_pvar_6()
+{
+ test("test_pvar_6 : prepared select into utext pointer");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=?");
+#line 798 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 798 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 798 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 800 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 800 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 800 "utext.pgc"
+
+ print_utext(p_utext_var);
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 804 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 804 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 804 "utext.pgc"
+
+ print_utext(p_utext_var);
+ init_table_value();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 808 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 808 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 808 "utext.pgc"
+
+
+}
+
+//prepared select using utext pointer
+void test_pvar_7()
+{
+ test("test_pvar_7 : prepared select using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 818 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 818 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 818 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 820 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 820 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 820 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 821 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 821 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 821 "utext.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 824 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 824 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 824 "utext.pgc"
+
+}
+
+//prepared update using utext pointer
+void test_pvar_8()
+{
+ test("test_pvar_8 : prepared update using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 833 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 833 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 833 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 835 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 835 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 835 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 836 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 836 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 836 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 837 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 837 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 837 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 838 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 838 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 838 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 841 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 841 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 841 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using utext pointer
+void test_pvar_9()
+{
+ test("test_pvar_9 : prepared delete using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 852 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 852 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 852 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 854 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 854 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 854 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 855 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 855 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 855 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 856 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 856 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 856 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 859 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 859 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 859 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using utext pointer
+void test_pvar_10()
+{
+ test("test_pvar_10 : prepared insert using utext pointer");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 870 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 870 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 870 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, 13)");
+#line 872 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 872 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 872 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 873 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 873 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 873 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 874 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 874 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 874 "utext.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 877 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 877 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 877 "utext.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using utext pointer
+void test_pvar_11()
+{
+ test("test_pvar_11 : Open cursor using utext pointer");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 888 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 888 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 888 "utext.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 889 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 889 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 889 "utext.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 891 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 891 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 891 "utext.pgc"
+
+ /* declare cursor_pvar_11 cursor for $1 */
+#line 892 "utext.pgc"
+
+ { ECPGopen("cursor_pvar_11", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_pvar_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 893 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 893 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 893 "utext.pgc"
+
+ { ECPGfetch("cursor_pvar_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_pvar_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 894 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 894 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 894 "utext.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGclose("cursor_pvar_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_pvar_11", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 896 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 896 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 896 "utext.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 897 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 897 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 897 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 898 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 898 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 898 "utext.pgc"
+
+}
+
+//Fecth cursor into utext pointer
+void test_pvar_12()
+{
+ test("test_pvar_12 : Fecth cursor into utext pointer");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 907 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 907 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 907 "utext.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=1");
+#line 908 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 908 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 908 "utext.pgc"
+
+ /* declare cursor_pvar_12 cursor for $1 */
+#line 909 "utext.pgc"
+
+ { ECPGopen("cursor_pvar_12", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_pvar_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 910 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 910 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 910 "utext.pgc"
+
+ { ECPGfetch("cursor_pvar_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_pvar_12", ECPGt_EOIT,
+ ECPGt_utext,&(p_utext_var),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 911 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 911 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 911 "utext.pgc"
+
+ { ECPGclose("cursor_pvar_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_pvar_12", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 912 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 912 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 912 "utext.pgc"
+
+
+ print_utext(p_utext_var);
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 916 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 916 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 916 "utext.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 917 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 917 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 917 "utext.pgc"
+
+}
+
+//simple insert utext pointer with L string inited
+void test_pvar_13()
+{
+/* exec sql begin declare section */
+
+
+#line 924 "utext.pgc"
+ utext * utext_local_pvar = L"太𠮷" ;
+/* exec sql end declare section */
+#line 925 "utext.pgc"
+
+ test("test_pvar_13 : simple insert utext pointer with L string inited");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 16 )",
+ ECPGt_utext,&(utext_local_pvar),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 929 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 929 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 929 "utext.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 16", ECPGt_EOIT,
+ ECPGt_utext,(utext_var),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_int,&(utext_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 931 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 931 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 931 "utext.pgc"
+
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+ init_table_value();
+}
+
+//Buck insert utext array into table
+void test_buckinsert_1()
+{
+ test("test_buckinsert_1 : Buck insert utext array into table");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 944 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 944 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 944 "utext.pgc"
+
+
+ // truncate table
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "truncate tb1", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 947 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 947 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 947 "utext.pgc"
+
+
+ //buck insert
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "BULK_INSERT_V01_I0058R0000 insert into tb1 (Item , count) values ( $1 , $2 )",
+ ECPGt_utext,(utext_array),(long)VAR_SIZE,(long)ARRAY_SIZE,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_int,(count_array),(long)1,(long)4,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT,
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOLT);
+#line 950 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 950 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 950 "utext.pgc"
+
+
+ //check the results
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1", ECPGt_EOIT,
+ ECPGt_int,&(total_tuples),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 953 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 953 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 953 "utext.pgc"
+
+ printf ("Total tuples in tb1 = %d\n", total_tuples);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[0]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 956 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 956 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 956 "utext.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_utext,(utext_array[2]),(long)VAR_SIZE,(long)1,(VAR_SIZE)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 959 "utext.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 959 "utext.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 959 "utext.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_var();
+ init_table_value();
+}
+
+void test_all()
+{
+ test_var_1();
+ test_var_2();
+ test_var_3();
+ test_var_4();
+ test_var_5();
+ test_var_6();
+ test_var_7();
+ test_var_8();
+ test_var_9();
+ test_var_10();
+ test_var_11();
+ test_var_12();
+ test_var_13();
+ test_array_1();
+ test_array_2();
+ test_array_3();
+ test_array_4();
+ test_array_5();
+ test_array_6();
+ test_array_7();
+ test_array_8();
+ test_array_9();
+ test_array_10();
+ test_array_11();
+ test_array_12();
+ test_array_13();
+ test_pvar_1();
+ test_pvar_2();
+ test_pvar_3();
+ test_pvar_4();
+ test_pvar_5();
+ test_pvar_6();
+ test_pvar_7();
+ test_pvar_8();
+ test_pvar_9();
+ test_pvar_10();
+ test_pvar_11();
+ test_pvar_12();
+ test_pvar_13();
+ test_buckinsert_1();
+}
+
+int main(int argc, char *argv[])
+{
+ ECPGdebug(1, stderr);
+ if(test_init() !=0)
+ return -1;
+
+ test_all();
+ //test_var_13();
+ //test_array_13();
+ //test_pvar_13();
+
+ test_finish();
+
+ return 0;
+}
\ No newline at end of file
diff --git a/src/interfaces/ecpg/test/unicode/expected/utext.ret b/src/interfaces/ecpg/test/unicode/expected/utext.ret
new file mode 100644
index 0000000..19088ec
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/expected/utext.ret
@@ -0,0 +1,435 @@
+SQL error:
+
+test_var_1: simple select into utext var
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E9A 0x00006D32 0x00006B27 0x00006D32 0x0000975E 0x00006D32 0x00005927 0x00006D0B
+0x00006D32 0x00005317 0x00007F8E 0x00006D32 0x00005357 0x00007F8E 0x00006D32 0x00005357
+0x00006781 0x00006D32 0x00006CA1 0x00006709
+======End utext_var content======
+utext_var_ind = 23
+
+test_var_2: simple select using utext var
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_3: simple update using utext var
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_var_4 : simple delete using utext var
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_5 : simple insert using utext
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_6 : prepared select into utext var
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+
+test_var_7 : prepared select using utext var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_8 : prepared update using utext var
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_9 : prepared delete using utext var
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_10 : prepared insert using utext var
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_11 : Open cursor using utext var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_12 : Fecth cursor into utext var
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+
+test_var_13 : simple insert utext var with L string inited
+======print utext_var_str content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403
+======End utext_var_str content======
+======print utext_var content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+
+test_var_14 : Open cursor using utext var directly in WHERE Clause
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_15 : Test utext_var working with NULL without indicator
+======print utext_var content======
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+found 6 rows for Item being NULL
+
+test_var_16 : Test utext_var working with NULL with indicator
+======print utext_var content======
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_array_1 : simple select into utext array
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+---->array[3]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[2]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[3]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_2 : simple select using array
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_3 : simple update using utext array
+find 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+find 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_4 : simple delete using utext array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_5 : simple insert using utext array
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_6 : prepared select SQL error:
+into utext array
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+---->array[3]:
+0x00004E9A 0x00006D32 0x00006B27 0x00006D32 0x0000975E 0x00006D32 0x00005927 0x00006D0B
+0x00006D32 0x00005317 0x00007F8E 0x00006D32 0x00005357 0x00007F8E 0x00006D32 0x00005357
+0x00006781 0x00006D32 0x00006CA1 0x00006709
+
+
+test_array_7 : prepared select using utext array
+count=1 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_8 : prepared update using utext array
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_9 : prepared delete using utext array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_10 : prepared insert using utext array
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_11 : Open cursor using utext array
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_12 : Fecth cursor into utext array
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+---->array[3]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_13 : Insert array with L string inited
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+
+
+test_array_14 : Open cursor using utext array directly in WHERE Clause
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_15 : Test utext array working with NULL without using indicator
+---->array[0]:
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[1]:
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[2]:
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+found 6 rows for Item being NULL
+
+test_array_16 : Test utext array working with NULL using indicator
+---->array[0]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[1]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[2]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+utext_var_ind = -1
+utext_var_ind = -1
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_pvar_1 : simple select into utext pointer
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E9A 0x00006D32 0x00006B27 0x00006D32 0x0000975E 0x00006D32 0x00005927 0x00006D0B
+0x00006D32 0x00005317 0x00007F8E 0x00006D32 0x00005357 0x00007F8E 0x00006D32 0x00005357
+0x00006781 0x00006D32 0x00006CA1 0x00006709
+======End utext_var content======
+utext_var_ind = 0
+
+test_pvar_2 : simple select using utext pointer
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=0 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_3 : simple update using utext pointer
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_pvar_4 : simple delete using utext pointer
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_5 : simple insert using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_6 : prepared select into utext pointer
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+
+test_pvar_7 : prepared select using utext pointer
+count=0 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_8 : prepared update using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_9 : prepared delete using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_10 : prepared insert using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_11 : Open cursor using utext pointer
+count=0 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_12 : Fecth cursor into utext pointer
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+
+test_pvar_13 : simple insert utext pointer with L string inited
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+
+test_pvar_14 : Open cursor using utext var directly in WHERE Clause
+count=0 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_15 : Test utext pointer working with NULL without using indicator
+======print utext_var content======
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+found 6 rows for Item being NULL
+
+test_pvar_16 : Test utext pointer working with NULL using indicator
+======print utext_var content======
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_pvar_17 : Test utext uninitialized pointer getting and setting data
+======print utext_var_str content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B
+======End utext_var_str content======
+Find Item = 太𠮷𠜱平洋𠱓大西洋印度洋北冰洋 where Count=18
+
+test_pvar_18 : Test utext uninitialized pointer working with NULL without using indicator
+======print utext_var content======
+0x00000000
+======End utext_var content======
+found 6 rows for Item being NULL
+
+test_pvar_19 : Test utext uninitialized pointer working with NULL using indicator
+======print utext_var content======
+0x00000000
+======End utext_var content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_pp_var_1 : Test pointer to pointer utext_var without initialize
+======print utext_var_str content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B
+======End utext_var_str content======
+utext_var_ind = 0
+======print utext_var_str content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403
+======End utext_var_str content======
+utext_var_ind = 0
+======print utext_var_str content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F
+======End utext_var_str content======
+utext_var_ind = 0
+足球篮球羽毛球乒乓球橄榄球棒球冰球
+
+test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator
+======print utext_var_str content======
+
+======End utext_var_str content======
+======print utext_var_str content======
+
+======End utext_var_str content======
+======print utext_var_str content======
+
+======End utext_var_str content======
+found 6 rows for Item being NULL
+
+test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator
+======print utext_var_str content======
+
+======End utext_var_str content======
+utext_var_ind = -1
+======print utext_var_str content======
+
+======End utext_var_str content======
+utext_var_ind = -1
+======print utext_var_str content======
+
+======End utext_var_str content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
diff --git a/src/interfaces/ecpg/test/unicode/expected/utext_ex.c b/src/interfaces/ecpg/test/unicode/expected/utext_ex.c
new file mode 100755
index 0000000..f4e44aa
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/expected/utext_ex.c
@@ -0,0 +1,521 @@
+/* Processed by ecpg (regression mode) */
+/* These include files are added by the preprocessor */
+#define ECPG_ENABLE_UTEXT 1
+#include <ecpglib.h>
+#include <ecpgerrno.h>
+#include <sqlca.h>
+/* End of automatic include section */
+#define ECPGdebug(X,Y) ECPGdebug((X)+100,(Y))
+
+#line 1 "utext_ex.pgc"
+#include <stdio.h>
+
+
+#line 1 "./../regression.h"
+
+
+
+
+
+
+#line 3 "utext_ex.pgc"
+
+
+void print_utext(char *var_name, utext *utext_var, int var_size);
+void print_utext_array(char *var_name, utext *utext_var, int var_size, int index);
+void print_nchar(char *utext_var, int var_size);
+
+#define print_ret print_utext("employee",employee,40); \
+print_utext("address", address.arr, address.len)
+
+/* exec sql begin declare section */
+
+
+
+
+
+
+
+
+
+
+
+
+#line 13 "utext_ex.pgc"
+ char char_ename [ 3 ] [ 40 ] = { "test ok" , "测试 通过" , "太𠮷𠜱" } ;
+
+#line 15 "utext_ex.pgc"
+ char employee_1 [ 40 ] = "test ok" ;
+
+#line 16 "utext_ex.pgc"
+ char employee_2 [ 40 ] = "测试 通过" ;
+
+#line 17 "utext_ex.pgc"
+ char employee_3 [ 40 ] = "太𠮷𠜱" ;
+
+#line 19 "utext_ex.pgc"
+ char char_addr [ 3 ] [ 40 ] = { "1 sydney, NSW" , "澳大利亚悉尼1号" , "世界杯" } ;
+
+#line 21 "utext_ex.pgc"
+ char address_1 [ 40 ] = "1 sydney, NSW" ;
+
+#line 22 "utext_ex.pgc"
+ char address_2 [ 40 ] = "澳大利亚悉尼1号" ;
+
+#line 23 "utext_ex.pgc"
+ char address_3 [ 40 ] = "世界杯" ;
+/* exec sql end declare section */
+#line 24 "utext_ex.pgc"
+
+
+void print_utext(char *var_name, utext *utext_var, int var_size)
+{
+ int i;
+ printf("======print %s content======\n",var_name);
+ for(i=0; i<var_size; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_utext_array(char *var_name, utext *utext_var, int var_size, int index)
+{
+ int i;
+ printf("======print %s[%d] content======\n",var_name,index);
+ for(i=0; i<var_size; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_nchar(char *utext_var, int var_size)
+{
+ printf("nchar is : %s\n",utext_var);
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<var_size; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void create_table()
+{
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "create table if not exists emp ( id int , ename char ( 20 ) , address nvarchar ( 50 ) )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 68 "utext_ex.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "truncate table emp", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 69 "utext_ex.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "create table if not exists emp_bk ( id int , ename char ( 20 ) , address nvarchar ( 50 ) )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 71 "utext_ex.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "truncate table emp_bk", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 72 "utext_ex.pgc"
+
+}
+
+void init_table()
+{
+ /* UTF8 is the database character set */
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into emp ( id , ename , address ) values ( 1 , 'test ok' , '1 sydney, NSW' )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 78 "utext_ex.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into emp ( id , ename , address ) values ( 2 , '测试 通过' , '澳大利亚悉尼1号' )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 79 "utext_ex.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into emp ( id , ename , address ) values ( 3 , '太𠮷𠜱' , '世界杯' )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 80 "utext_ex.pgc"
+
+}
+
+/*
+ Test utext host variable
+*/
+void testcase1()
+{
+ /* exec sql begin declare section */
+
+
+
+
+ /* define Unicode host variable */
+ /* define a variable length Unicode host variable */
+
+#line 89 "utext_ex.pgc"
+ int total_tuples = 0 ;
+
+#line 90 "utext_ex.pgc"
+ int i = 0 ;
+
+#line 91 "utext_ex.pgc"
+ int id = 0 ;
+
+#line 92 "utext_ex.pgc"
+ int id2 = 0 ;
+
+#line 93 "utext_ex.pgc"
+ utext employee [ 41 ] ;
+
+#line 94 "utext_ex.pgc"
+ struct uvarchar_1 { int len; utext arr[ 101 ]; } address ;
+/* exec sql end declare section */
+#line 95 "utext_ex.pgc"
+
+
+ printf("\n**************Start testcase(%d)**************\n",1);
+
+ memset(employee,'a',sizeof(employee));
+ memset((void*)&address,'a',sizeof(address));
+ address.len = 0;
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "truncate emp_bk", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 103 "utext_ex.pgc"
+
+
+ for(i=1;i<=3;i++)
+ {
+ /* Database character set converted to Unicode */
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select ename , address from emp where id = $1 ",
+ ECPGt_int,&(i),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,(employee),(long)41,(long)1,(41)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address),(long)101,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 108 "utext_ex.pgc"
+
+ print_ret;
+ /* Unicode converted to Database character */
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into emp_bk ( id , ename , address ) values ( $1 , $2 , $3 )",
+ ECPGt_int,&(i),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_utext,(employee),(long)41,(long)1,(41)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address),(long)101,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 111 "utext_ex.pgc"
+
+ }
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from emp_bk", ECPGt_EOIT,
+ ECPGt_int,&(total_tuples),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 114 "utext_ex.pgc"
+
+ printf("total_tuples = %d in emp_bk\n",total_tuples);
+
+
+ for(i=3;i>0;i--)
+ {
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select ename , address from emp where id = $1 ",
+ ECPGt_int,&(i),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,(employee),(long)41,(long)1,(41)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address),(long)101,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 120 "utext_ex.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select id from emp_bk where ename = $1 and address = $2 ",
+ ECPGt_utext,(employee),(long)41,(long)1,(41)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address),(long)101,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(id),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 121 "utext_ex.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select id from emp_bk where ename = $1 and address = $2 ",
+ ECPGt_char,(char_ename[i-1]),(long)40,(long)1,(40)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_char,(char_addr[i-1]),(long)40,(long)1,(40)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(id2),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 122 "utext_ex.pgc"
+
+ printf("id[%d] = id2[%d] in emp_bk\n",id,id2);
+ }
+}
+
+/*
+ Test utext array
+*/
+void testcase2()
+{
+ /* exec sql begin declare section */
+
+
+
+
+
+ /* define Unicode host variable */
+ /* define a variable length Unicode host variable */
+
+#line 133 "utext_ex.pgc"
+ int total_tuples = 0 ;
+
+#line 134 "utext_ex.pgc"
+ int i = 0 ;
+
+#line 135 "utext_ex.pgc"
+ int j = 0 ;
+
+#line 136 "utext_ex.pgc"
+ int id = 0 ;
+
+#line 137 "utext_ex.pgc"
+ int id2 = 0 ;
+
+#line 138 "utext_ex.pgc"
+ utext employee_array [ 3 ] [ 41 ] ;
+
+#line 139 "utext_ex.pgc"
+ struct uvarchar_2 { int len; utext arr[ 101 ]; } address_array [ 3 ] ;
+/* exec sql end declare section */
+#line 140 "utext_ex.pgc"
+
+
+ printf("\n**************Start testcase(%d)**************\n",2);
+
+ memset(employee_array,'a',sizeof(employee_array));
+
+ for(i=0;i<3;i++)
+ {
+ memset((void*)&address_array[i],'a',sizeof(address_array[0]));
+ address_array[i].len=0;
+ }
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "truncate emp_bk", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 153 "utext_ex.pgc"
+
+
+ /* Database character set converted to Unicode */
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select ename , address from emp", ECPGt_EOIT,
+ ECPGt_utext,(employee_array),(long)41,(long)3,(41)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,(address_array),(long)101,(long)3,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 156 "utext_ex.pgc"
+
+
+ for(i=1;i<=3;i++)
+ {
+ print_utext_array("employee_array",employee_array[i-1],40,i-1);
+ print_utext_array("address_array",address_array[i-1].arr,address_array[i-1].len,i-1);
+ /* Unicode converted to Database character */
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into emp_bk ( id , ename , address ) values ( $1 , $2 , $3 )",
+ ECPGt_int,&(i),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_utext,(employee_array[i-1]),(long)41,(long)1,(41)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address_array[i-1]),(long)101,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 163 "utext_ex.pgc"
+
+ }
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from emp_bk", ECPGt_EOIT,
+ ECPGt_int,&(total_tuples),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 166 "utext_ex.pgc"
+
+ printf("total_tuples = %d in emp_bk\n",total_tuples);
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select ename , address from emp", ECPGt_EOIT,
+ ECPGt_utext,(employee_array),(long)41,(long)3,(41)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,(address_array),(long)101,(long)3,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 170 "utext_ex.pgc"
+
+ for(i=3;i>0;i--)
+ {
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select id from emp_bk where ename = $1 and address = $2 ",
+ ECPGt_utext,(employee_array[i-1]),(long)41,(long)1,(41)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address_array[i-1]),(long)101,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(id),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 173 "utext_ex.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select id from emp_bk where ename = $1 and address = $2 ",
+ ECPGt_char,(char_ename[i-1]),(long)40,(long)1,(40)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_char,(char_addr[i-1]),(long)40,(long)1,(40)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(id2),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 174 "utext_ex.pgc"
+
+ printf("id[%d] = id2[%d] in emp_bk\n",id,id2);
+ }
+}
+
+
+/*
+ Test utext pointer
+*/
+void testcase3()
+{
+ /* exec sql begin declare section */
+
+
+
+
+ /* define Unicode host variable */
+
+ /* define a variable length Unicode host variable */
+
+#line 186 "utext_ex.pgc"
+ int total_tuples = 0 ;
+
+#line 187 "utext_ex.pgc"
+ int i = 0 ;
+
+#line 188 "utext_ex.pgc"
+ int id = 0 ;
+
+#line 189 "utext_ex.pgc"
+ int id2 = 0 ;
+
+#line 190 "utext_ex.pgc"
+ utext * employee ;
+
+#line 191 "utext_ex.pgc"
+ int employee_len = 82 ;
+
+#line 192 "utext_ex.pgc"
+ struct uvarchar_3 { int len; utext arr[ 101 ]; } address ;
+/* exec sql end declare section */
+#line 193 "utext_ex.pgc"
+
+
+
+ printf("\n**************Start testcase(%d)**************\n",3);
+
+ employee = malloc(employee_len);
+
+ memset(employee,'a',employee_len);
+ memset((void*)&address,'a',sizeof(address));
+ address.len = 0;
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "truncate emp_bk", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 204 "utext_ex.pgc"
+
+
+ for(i=1;i<=3;i++)
+ {
+ /* Database character set converted to Unicode */
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select ename , address from emp where id = $1 ",
+ ECPGt_int,&(i),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,&(employee),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address),(long)101,(long)1,sizeof(struct uvarchar_3),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 209 "utext_ex.pgc"
+
+ print_ret;
+ /* Unicode converted to Database character */
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into emp_bk ( id , ename , address ) values ( $1 , $2 , $3 )",
+ ECPGt_int,&(i),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_utext,&(employee),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address),(long)101,(long)1,sizeof(struct uvarchar_3),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);}
+#line 212 "utext_ex.pgc"
+
+ }
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from emp_bk", ECPGt_EOIT,
+ ECPGt_int,&(total_tuples),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 215 "utext_ex.pgc"
+
+ printf("total_tuples = %d in emp_bk\n",total_tuples);
+
+
+ for(i=3;i>0;i--)
+ {
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select ename , address from emp where id = $1 ",
+ ECPGt_int,&(i),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_utext,&(employee),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address),(long)101,(long)1,sizeof(struct uvarchar_3),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 221 "utext_ex.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select id from emp_bk where ename = $1 and address = $2 ",
+ ECPGt_utext,&(employee),(long)0,(long)1,(1)*sizeof(utext),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(address),(long)101,(long)1,sizeof(struct uvarchar_3),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(id),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 222 "utext_ex.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select id from emp_bk where ename = $1 and address = $2 ",
+ ECPGt_char,(char_ename[i-1]),(long)40,(long)1,(40)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_char,(char_addr[i-1]),(long)40,(long)1,(40)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(id2),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);}
+#line 223 "utext_ex.pgc"
+
+ printf("id[%d] = id2[%d] in emp_bk\n",id,id2);
+ }
+}
+
+int main() {
+ { ECPGconnect(__LINE__, 0, "regress1" , NULL, NULL , NULL, 0); }
+#line 229 "utext_ex.pgc"
+
+
+ { ECPGsetcommit(__LINE__, "on", NULL);}
+#line 231 "utext_ex.pgc"
+
+ /* exec sql whenever sql_warning sqlprint ; */
+#line 232 "utext_ex.pgc"
+
+ /* exec sql whenever sqlerror sqlprint ; */
+#line 233 "utext_ex.pgc"
+
+ create_table();
+ init_table();
+
+ testcase1();
+ testcase2();
+ testcase3();
+
+ { ECPGdisconnect(__LINE__, "CURRENT");
+#line 241 "utext_ex.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 241 "utext_ex.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 241 "utext_ex.pgc"
+
+ return 0;
+}
+
+
+
diff --git a/src/interfaces/ecpg/test/unicode/expected/utext_ex.ret b/src/interfaces/ecpg/test/unicode/expected/utext_ex.ret
new file mode 100755
index 0000000..b2aaeb0
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/expected/utext_ex.ret
@@ -0,0 +1,105 @@
+
+**************Start testcase(1)**************
+======print employee content======
+0x0074 0x0065 0x0073 0x0074 0x0020 0x006F 0x006B 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+
+======print address content======
+0x0031 0x0020 0x0073 0x0079 0x0064 0x006E 0x0065 0x0079
+0x002C 0x0020 0x004E 0x0053 0x0057
+======print employee content======
+0x6D4B 0x8BD5 0x0020 0x901A 0x8FC7 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+
+======print address content======
+0x6FB3 0x5927 0x5229 0x4E9A 0x6089 0x5C3C 0x0031 0x53F7
+
+======print employee content======
+0x592A 0xD842 0xDFB7 0xD841 0xDF31 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+
+======print address content======
+0x4E16 0x754C 0x676F
+total_tuples = 3 in emp_bk
+id[3] = id2[3] in emp_bk
+id[2] = id2[2] in emp_bk
+id[1] = id2[1] in emp_bk
+
+**************Start testcase(2)**************
+======print employee_array[0] content======
+0x0074 0x0065 0x0073 0x0074 0x0020 0x006F 0x006B 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+
+======print address_array[0] content======
+0x0031 0x0020 0x0073 0x0079 0x0064 0x006E 0x0065 0x0079
+0x002C 0x0020 0x004E 0x0053 0x0057
+======print employee_array[1] content======
+0x6D4B 0x8BD5 0x0020 0x901A 0x8FC7 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+
+======print address_array[1] content======
+0x6FB3 0x5927 0x5229 0x4E9A 0x6089 0x5C3C 0x0031 0x53F7
+
+======print employee_array[2] content======
+0x592A 0xD842 0xDFB7 0xD841 0xDF31 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000 0x0000
+
+======print address_array[2] content======
+0x4E16 0x754C 0x676F
+total_tuples = 3 in emp_bk
+id[3] = id2[3] in emp_bk
+id[2] = id2[2] in emp_bk
+id[1] = id2[1] in emp_bk
+
+**************Start testcase(3)**************
+======print employee content======
+0x0074 0x0065 0x0073 0x0074 0x0020 0x006F 0x006B 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0000 0x6161 0x6161 0x6161
+0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161
+0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161
+
+======print address content======
+0x0031 0x0020 0x0073 0x0079 0x0064 0x006E 0x0065 0x0079
+0x002C 0x0020 0x004E 0x0053 0x0057
+======print employee content======
+0x6D4B 0x8BD5 0x0020 0x901A 0x8FC7 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0000 0x6161 0x6161 0x6161
+0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161
+0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161
+
+======print address content======
+0x6FB3 0x5927 0x5229 0x4E9A 0x6089 0x5C3C 0x0031 0x53F7
+
+======print employee content======
+0x592A 0xD842 0xDFB7 0xD841 0xDF31 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0020
+0x0020 0x0020 0x0020 0x0020 0x0020 0x0020 0x0000 0x6161
+0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161
+0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161 0x6161
+
+======print address content======
+0x4E16 0x754C 0x676F
+total_tuples = 3 in emp_bk
+id[3] = id2[3] in emp_bk
+id[2] = id2[2] in emp_bk
+id[1] = id2[1] in emp_bk
diff --git a/src/interfaces/ecpg/test/unicode/expected/uvarchar.c b/src/interfaces/ecpg/test/unicode/expected/uvarchar.c
new file mode 100755
index 0000000..be42b60
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/expected/uvarchar.c
@@ -0,0 +1,2195 @@
+/* Processed by ecpg (regression mode) */
+/* These include files are added by the preprocessor */
+#define ECPG_ENABLE_UTEXT 1
+#include <ecpglib.h>
+#include <ecpgerrno.h>
+#include <sqlca.h>
+/* End of automatic include section */
+#define ECPGdebug(X,Y) ECPGdebug((X)+100,(Y))
+
+#line 1 "uvarchar.pgc"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+#line 1 "./../regression.h"
+
+
+
+
+
+
+#line 5 "uvarchar.pgc"
+
+
+#define test(msg) printf("\n%s\n",msg)
+#define VAR_SIZE 20
+#define ARRAY_SIZE 4
+
+/* Following is UTF8 and UTF16 characters mapping table */
+/*太𠮷𠜱平洋𠱓大西洋印度洋北冰洋
+0x592A,0x5E73,0x6D0B,0x5927,0x897F,0x6D0B,0x5370,0x5EA6,
+0x6D0B,0x5317,0x51B0,0x6D0B,0x548C,0x5357,0x51B0,0x6D0B,
+0x0000,0x0000,0x0000,0x0000*/
+
+/*足球篮球羽毛球乒乓球橄榄球棒球冰球
+0x8DB3,0x7403,0x7BEE,0x7403,0x7FBD,0x6BDB,0x7403,0x4E52,
+0x4E53,0x7403,0x6A44,0x6984,0x7403,0x68D2,0x7403,0x51B0,
+0x7403,0x0000,0x0000,0x0000
+*/
+
+/*世界杯每隔四年就会举行一次每次𠲖个球队
+0x4E16,0x754C,0x676F,0x6BCF,0x9694,0x56DB,0x5E74,0x5C31,
+0x4F1A,0x4E3E,0x884C,0x4E00,0x6B21,0x6BCF,0x6B21,0x0033,
+0x0032,0x4E2A,0x7403,0x961F
+*/
+
+/* 亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲
+0x4E9A,0x6D32,0x6B27,0x6D32,0x975E,0x6D32,0x5927,0x6D0B,
+0x6D32,0x5317,0x7F8E,0x6D32,0x5357,0x7F8E,0x6D32,0x5357,
+0x6781,0x6D32,0x6CA1,0x6709,0x5317,0x6781,0x6D32
+*/
+
+/* exec sql begin declare section */
+
+
+
+
+
+
+
+
+
+#line 36 "uvarchar.pgc"
+ struct uvarchar_1 { int len; utext arr[ VAR_SIZE ]; } uvarchar_var ;
+
+#line 37 "uvarchar.pgc"
+ struct uvarchar_2 { int len; utext arr[ VAR_SIZE ]; } uvarchar_array [ ARRAY_SIZE ] ;
+
+#line 39 "uvarchar.pgc"
+ int uvarchar_var_ind ;
+
+#line 41 "uvarchar.pgc"
+ int count ;
+
+#line 42 "uvarchar.pgc"
+ int count_array [ 4 ] = { 1 , 2 , 3 , 4 } ;
+
+#line 43 "uvarchar.pgc"
+ int total_tuples = 0 ;
+/* exec sql end declare section */
+#line 44 "uvarchar.pgc"
+
+
+void print_uvarchar(void);
+void print_uvarchar_ind(int uvarchar_var_ind);
+void print_array(void);
+int test_init(void);
+void test_finish(void);
+void init_table_value(void);
+void init_var(void);
+
+void test_var_1(void);
+void test_var_2(void);
+void test_var_3(void);
+void test_var_4(void);
+void test_var_5(void);
+void test_var_6(void);
+void test_var_7(void);
+void test_var_8(void);
+void test_var_9(void);
+void test_var_10(void);
+void test_var_11(void);
+void test_var_12(void);
+void test_var_13(void);
+void test_array_1(void);
+void test_array_2(void);
+void test_array_3(void);
+void test_array_4(void);
+void test_array_5(void);
+void test_array_6(void);
+void test_array_7(void);
+void test_array_8(void);
+void test_array_9(void);
+void test_array_10(void);
+void test_array_11(void);
+void test_array_12(void);
+void test_array_13(void);
+void test_buckinsert_1(void);
+
+void test_all(void);
+
+void print_uvarchar()
+{
+ int i;
+
+ printf ("---->uvarchar variable,len=%d:\n",uvarchar_var.len);
+ for(i=0; i<VAR_SIZE; i++)
+ {
+ printf ("0x%04X ", uvarchar_var.arr[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_uvarchar_ind(int uvarchar_var_ind)
+{
+ printf("uvarchar_var_ind = %d\n",uvarchar_var_ind);
+}
+
+void print_array()
+{
+ int i,j;
+
+ for(i=0; i<ARRAY_SIZE; i++)
+ {
+ printf ("---->uvarchar array[%d]:(len=%d)\n", i,uvarchar_array[i].len);
+
+ for(j=0; j<VAR_SIZE; j++)
+ {
+ printf ("0x%04X ", uvarchar_array[i].arr[j]);
+ if(j>6 && (j+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+ }
+
+ printf("\n");
+}
+
+int test_init()
+{
+ { ECPGconnect(__LINE__, 0, "regress1" , NULL, NULL , NULL, 0); }
+#line 125 "uvarchar.pgc"
+
+
+ { ECPGsetcommit(__LINE__, "on", NULL);}
+#line 127 "uvarchar.pgc"
+
+ /* exec sql whenever sql_warning sqlprint ; */
+#line 128 "uvarchar.pgc"
+
+ /* exec sql whenever sqlerror sqlprint ; */
+#line 129 "uvarchar.pgc"
+
+
+ //initialization of test table
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "create table tb1 ( Item varchar , count integer )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 132 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 132 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 132 "uvarchar.pgc"
+
+
+ init_table_value();
+
+ return 0;
+}
+
+void test_finish()
+{
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "drop table tb1", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 141 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 141 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 141 "uvarchar.pgc"
+
+ { ECPGdisconnect(__LINE__, "ALL");
+#line 142 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 142 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 142 "uvarchar.pgc"
+
+}
+
+
+void init_table_value()
+{
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "truncate tb1", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 148 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 148 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 148 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋' , 1 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 150 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 150 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 150 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '足球篮球羽毛球乒乓球橄榄球棒球冰球' , 2 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 151 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 151 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 151 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '世界杯每隔四年就会举行一次每次𠲖个球队' , 3 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 152 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 152 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 152 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( '亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲' , 4 )", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 153 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 153 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 153 "uvarchar.pgc"
+
+}
+
+void init_var()
+{
+ int i;
+ memset((void*)&uvarchar_var,'a',sizeof(uvarchar_var));
+ uvarchar_var.len = 0;
+
+ memset((char*)uvarchar_array,'a',sizeof(uvarchar_array)*ARRAY_SIZE);
+
+ for(i=0;i<ARRAY_SIZE;i++)
+ {
+ uvarchar_array[i].len = 0;
+ }
+
+ uvarchar_var_ind = 0;
+}
+
+//simple select into uvarchar
+void test_var_1()
+{
+ test("test_var_1 : simple select into uvarchar var");
+ init_var();
+ //print_uvarchar();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 179 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 179 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 179 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 2", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 185 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 185 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 185 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 190 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 190 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 190 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 4", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 195 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 195 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 195 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+}
+
+
+//simple select using uvarchar
+void test_var_2()
+{
+ test("test_var_2 : simple select using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 208 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 208 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 208 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 209 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 209 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 209 "uvarchar.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 212 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 212 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 212 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 213 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 213 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 213 "uvarchar.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using uvarchar
+void test_var_3()
+{
+ test("test_var_3 : simple update using uvarchar");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 223 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 223 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 223 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 2",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 224 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 224 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 224 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 3",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 225 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 225 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 225 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 226 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 226 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 226 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using uvarchar var
+void test_var_4()
+{
+ test("test_var_4 : simple delete using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 238 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 238 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 238 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 239 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 239 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 239 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 240 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 240 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 240 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 243 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 243 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 243 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 244 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 244 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 244 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 245 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 245 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 245 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using uvarchar var
+void test_var_5()
+{
+ test("test_var_5 : simple insert using uvarchar");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 257 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 257 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 257 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 11 )",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 258 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 258 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 258 "uvarchar.pgc"
+
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 261 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 261 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 261 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 262 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 262 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 262 "uvarchar.pgc"
+
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 265 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 265 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 265 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 268 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 268 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 268 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into uvarchar var
+void test_var_6()
+{
+ test("test_var_6 : prepared select into uvarchar var");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=?");
+#line 280 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 280 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 280 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 282 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 282 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 282 "uvarchar.pgc"
+
+ print_uvarchar();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 285 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 285 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 285 "uvarchar.pgc"
+
+ print_uvarchar();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 288 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 288 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 288 "uvarchar.pgc"
+
+}
+
+//prepared select using uvarchar var
+void test_var_7()
+{
+ test("test_var_7 : prepared select using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 297 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 297 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 297 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 299 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 299 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 299 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 300 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 300 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 300 "uvarchar.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 303 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 303 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 303 "uvarchar.pgc"
+
+}
+
+//prepared update using uvarchar var
+void test_var_8()
+{
+ test("test_var_8 : prepared update using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 312 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 312 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 312 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 314 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 314 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 314 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 315 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 315 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 315 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 316 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 316 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 316 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 317 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 317 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 317 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 320 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 320 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 320 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using uvarchar var
+void test_var_9()
+{
+ test("test_var_9 : prepared delete using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 331 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 331 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 331 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 333 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 333 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 333 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 334 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 334 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 334 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 335 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 335 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 335 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 338 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 338 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 338 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using uvarchar var
+void test_var_10()
+{
+ test("test_var_10 : prepared insert using uvarchar var");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 349 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 349 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 349 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, 13)");
+#line 351 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 351 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 351 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 352 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 352 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 352 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 353 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 353 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 353 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 356 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 356 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 356 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using uvarchar var
+void test_var_11()
+{
+ test("test_var_11 : Open cursor using uvarchar var");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 367 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 367 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 367 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 3", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 368 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 368 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 368 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 370 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 370 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 370 "uvarchar.pgc"
+
+ /* declare cursor_var_11 cursor for $1 */
+#line 371 "uvarchar.pgc"
+
+ { ECPGopen("cursor_var_11", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_var_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 372 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 372 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 372 "uvarchar.pgc"
+
+ { ECPGfetch("cursor_var_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_var_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 373 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 373 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 373 "uvarchar.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGclose("cursor_var_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_var_11", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 375 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 375 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 375 "uvarchar.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 376 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 376 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 376 "uvarchar.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 377 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 377 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 377 "uvarchar.pgc"
+
+}
+
+//Fecth cursor into uvarchar var
+void test_var_12()
+{
+ test("test_var_12 : Fecth cursor into uvarchar var");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 386 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 386 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 386 "uvarchar.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count=1");
+#line 387 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 387 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 387 "uvarchar.pgc"
+
+ /* declare cursor_var_12 cursor for $1 */
+#line 388 "uvarchar.pgc"
+
+ { ECPGopen("cursor_var_12", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_var_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 389 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 389 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 389 "uvarchar.pgc"
+
+ { ECPGfetch("cursor_var_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_var_12", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 390 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 390 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 390 "uvarchar.pgc"
+
+ { ECPGclose("cursor_var_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_var_12", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 391 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 391 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 391 "uvarchar.pgc"
+
+
+ print_uvarchar();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 395 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 395 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 395 "uvarchar.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 396 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 396 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 396 "uvarchar.pgc"
+
+}
+
+//simple insert using uvarchar with L string inited
+void test_var_13()
+{
+/* exec sql begin declare section */
+
+
+#line 403 "uvarchar.pgc"
+ struct uvarchar_3 { int len; utext arr[ VAR_SIZE ]; } uvarchar_local_var ;
+/* exec sql end declare section */
+#line 404 "uvarchar.pgc"
+
+
+ test("test_var_13 : simple insert using uvarchar with L string inited");
+ init_var();
+
+ memset((char*)&uvarchar_local_var,0,sizeof(uvarchar_local_var));
+
+ memcpy((char*)uvarchar_local_var.arr, L"太𠮷𠜱", sizeof(L"太𠮷𠜱"));
+ uvarchar_local_var.len = sizeof(L"太𠮷𠜱")/2;
+
+ printf("uvarchar_local_var.len = %d\n",uvarchar_local_var.len);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 16 )",
+ ECPGt_uvarchar,&(uvarchar_local_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_3),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 416 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 416 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 416 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 418 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 418 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 418 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱'\n", count);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 16", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_var),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_1),
+ ECPGt_int,&(uvarchar_var_ind),(long)1,(long)1,sizeof(int), ECPGt_EORT, ECPGt_EOLT);
+#line 421 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 421 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 421 "uvarchar.pgc"
+
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+
+ init_table_value();
+}
+
+//simple select into uvarchar array
+void test_array_1()
+{
+ test("test_array_1 : simple select into uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 434 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 434 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 434 "uvarchar.pgc"
+
+
+ print_array();
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count = 1", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 439 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 439 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 439 "uvarchar.pgc"
+
+ print_array();
+ init_var();
+}
+
+
+//simple select using uvarchar array
+void test_array_2()
+{
+ test("test_array_2 : simple select using uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 451 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 451 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 451 "uvarchar.pgc"
+
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 454 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 454 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 454 "uvarchar.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 458 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 458 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 458 "uvarchar.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using uvarchar array
+void test_array_3()
+{
+ test("test_array_3 : simple update using uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 468 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 468 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 468 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "update tb1 set Item = $1 where count = 1",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 470 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 470 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 470 "uvarchar.pgc"
+
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 473 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 473 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 473 "uvarchar.pgc"
+
+ printf ("find %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 477 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 477 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 477 "uvarchar.pgc"
+
+ printf ("find %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using uvarchar array
+void test_array_4()
+{
+ test("test_array_4 : simple delete using uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 489 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 489 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 489 "uvarchar.pgc"
+
+
+ count = 100;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 492 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 492 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 492 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 493 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 493 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 493 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 100;
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "delete from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 497 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 497 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 497 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 498 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 498 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 498 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using uvarchar array
+void test_array_5()
+{
+ test("test_array_5 : simple insert using uvarchar array");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 510 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 510 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 510 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 11 )",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 512 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 512 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 512 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 513 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 513 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 513 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 13 )",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 516 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 516 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 516 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 517 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 517 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 517 "uvarchar.pgc"
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into uvarchar array
+void test_array_6()
+{
+ test("test_array_6 : prepared select into uvarchar array");
+ init_var();
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count<=?");
+#line 529 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 529 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 529 "uvarchar.pgc"
+
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 531 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 531 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 531 "uvarchar.pgc"
+
+
+ print_array();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 535 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 535 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 535 "uvarchar.pgc"
+
+}
+
+//prepared select using uvarchar array
+void test_array_7()
+{
+ test("test_array_7 : prepared select using uvarchar array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 543 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 543 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 543 "uvarchar.pgc"
+
+
+ count = 0;
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 546 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 546 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 546 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 547 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 547 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 547 "uvarchar.pgc"
+
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 550 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 550 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 550 "uvarchar.pgc"
+
+}
+
+//prepared update using uvarchar array
+void test_array_8()
+{
+ test("test_array_8 : prepared update using uvarchar array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 558 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 558 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 558 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "UPDATE tb1 SET Item=? WHERE Count=?");
+#line 560 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 560 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 560 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"1",(long)1,(long)1,strlen("1"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 561 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 561 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 561 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"2",(long)1,(long)1,strlen("2"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 562 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 562 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 562 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 563 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 563 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 563 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 566 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 566 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 566 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//prepared delete using uvarchar array
+void test_array_9()
+{
+ test("test_array_9 : prepared delete using uvarchar array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 576 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 576 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 576 "uvarchar.pgc"
+
+
+ count = 0;
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "DELETE FROM tb1 WHERE Item=?");
+#line 579 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 579 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 579 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 580 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 580 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 580 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 581 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 581 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 581 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 584 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 584 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 584 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//prepared insert using uvarchar array
+void test_array_10()
+{
+ test("test_array_10 : prepared insert using uvarchar array");
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 594 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 594 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 594 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "INSERT INTO tb1 values (?, ?)");
+#line 596 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 596 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 596 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"13",(long)2,(long)1,strlen("13"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 597 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 597 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 597 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_execute, "stmt",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_const,"15",(long)2,(long)1,strlen("15"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 598 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 598 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 598 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1 where Item = '世界杯每隔四年就会举行一次每次𠲖个球队'", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 599 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 599 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 599 "uvarchar.pgc"
+
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 602 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 602 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 602 "uvarchar.pgc"
+
+
+ init_table_value();
+}
+
+//Open cursor using uvarchar array
+void test_array_11()
+{
+ test("test_array_11 : Open cursor using uvarchar array");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 613 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 613 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 613 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 614 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 614 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 614 "uvarchar.pgc"
+
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Count FROM tb1 WHERE Item=?");
+#line 616 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 616 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 616 "uvarchar.pgc"
+
+ /* declare cursor_array_11 cursor for $1 */
+#line 617 "uvarchar.pgc"
+
+ { ECPGopen("cursor_array_11", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_array_11 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 618 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 618 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 618 "uvarchar.pgc"
+
+ { ECPGfetch("cursor_array_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_array_11", ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 619 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 619 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 619 "uvarchar.pgc"
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ { ECPGclose("cursor_array_11", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_array_11", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 621 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 621 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 621 "uvarchar.pgc"
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 622 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 622 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 622 "uvarchar.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 623 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 623 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 623 "uvarchar.pgc"
+
+}
+
+//Fecth cursor into uvarchar array
+void test_array_12()
+{
+ test("test_array_12 : Fecth cursor into uvarchar array");
+ init_var();
+
+ { ECPGsetcommit(__LINE__, "off", NULL);
+#line 632 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 632 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 632 "uvarchar.pgc"
+
+ { ECPGprepare(__LINE__, NULL, 0, "stmt", "SELECT Item FROM tb1 WHERE Count<=3");
+#line 633 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 633 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 633 "uvarchar.pgc"
+
+ /* declare cursor_array_12 cursor for $1 */
+#line 634 "uvarchar.pgc"
+
+ { ECPGopen("cursor_array_12", "stmt", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "declare cursor_array_12 cursor for $1",
+ ECPGt_char_variable,(ECPGprepared_statement(NULL, "stmt", __LINE__)),(long)1,(long)1,(1)*sizeof(char),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 635 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 635 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 635 "uvarchar.pgc"
+
+ { ECPGfetch("cursor_array_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 636 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 636 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 636 "uvarchar.pgc"
+
+ { ECPGfetch("cursor_array_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_array[1]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 637 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 637 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 637 "uvarchar.pgc"
+
+ { ECPGfetch("cursor_array_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "fetch cursor_array_12", ECPGt_EOIT,
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 638 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 638 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 638 "uvarchar.pgc"
+
+ { ECPGclose("cursor_array_12", __LINE__, 0, 1, NULL, 0, ECPGst_normal, "close cursor_array_12", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 639 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 639 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 639 "uvarchar.pgc"
+
+
+ print_array();
+
+ { ECPGdeallocate(__LINE__, 0, NULL, "stmt");
+#line 643 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 643 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 643 "uvarchar.pgc"
+
+ { ECPGsetcommit(__LINE__, "on", NULL);
+#line 644 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 644 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 644 "uvarchar.pgc"
+
+}
+
+//Insert array with L string inited
+void test_array_13()
+{
+/* exec sql begin declare section */
+
+
+#line 651 "uvarchar.pgc"
+ struct uvarchar_4 { int len; utext arr[ VAR_SIZE ]; } uvarchar_array13 [ 3 ] ;
+/* exec sql end declare section */
+#line 652 "uvarchar.pgc"
+
+ test("test_array_13 : Insert array with L string inited");
+
+ memset((char*)&uvarchar_array13,0,sizeof(uvarchar_array13));
+
+ memcpy((char*)uvarchar_array13[0].arr, L"太𠮷", sizeof(L"太𠮷"));
+ uvarchar_array13[0].len = sizeof(L"太𠮷")/2;
+ memcpy((char*)uvarchar_array13[1].arr, L"𠮷太", sizeof(L"𠮷太"));
+ uvarchar_array13[1].len = sizeof(L"𠮷太")/2;
+ memcpy((char*)uvarchar_array13[2].arr, L"太平", sizeof(L"太平"));
+ uvarchar_array13[2].len = sizeof(L"太平")/2;
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 20 )",
+ ECPGt_uvarchar,&(uvarchar_array13[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_4),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 664 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 664 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 664 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 21 )",
+ ECPGt_uvarchar,&(uvarchar_array13[1]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_4),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 665 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 665 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 665 "uvarchar.pgc"
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "insert into tb1 values ( $1 , 22 )",
+ ECPGt_uvarchar,&(uvarchar_array13[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_4),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 666 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 666 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 666 "uvarchar.pgc"
+
+
+ init_var();
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 22 and count >= 20", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 669 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 669 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 669 "uvarchar.pgc"
+
+ print_array();
+ init_table_value();
+
+}
+
+//Buck insert uvarchar array into table
+void test_buckinsert_1()
+{
+ test("test_buckinsert_1 : Buck insert uvarchar array into table");
+ init_var();
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select Item from tb1 where count <= 3", ECPGt_EOIT,
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 681 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 681 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 681 "uvarchar.pgc"
+
+
+ // truncate table
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "truncate tb1", ECPGt_EOIT, ECPGt_EORT, ECPGt_EOLT);
+#line 684 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 684 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 684 "uvarchar.pgc"
+
+
+ //buck insert
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "BULK_INSERT_V01_I0058R0000 insert into tb1 (Item , count) values ( $1 , $2 )",
+ ECPGt_uvarchar,(uvarchar_array),(long)VAR_SIZE,(long)ARRAY_SIZE,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L,
+ ECPGt_int,(count_array),(long)1,(long)4,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT, ECPGt_EORT,
+ ECPGt_const,"3",(long)1,(long)1,strlen("3"),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOLT);
+#line 687 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 687 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 687 "uvarchar.pgc"
+
+
+ //check the results
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count ( * ) from tb1", ECPGt_EOIT,
+ ECPGt_int,&(total_tuples),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 690 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 690 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 690 "uvarchar.pgc"
+
+ printf ("Total tuples in tb1 = %d\n", total_tuples);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[0]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 693 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 693 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 693 "uvarchar.pgc"
+
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ { ECPGdo(__LINE__, 0, 1, NULL, 0, ECPGst_normal, "select count from tb1 where Item = $1 ",
+ ECPGt_uvarchar,&(uvarchar_array[2]),(long)VAR_SIZE,(long)1,sizeof(struct uvarchar_2),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EOIT,
+ ECPGt_int,&(count),(long)1,(long)1,sizeof(int),
+ ECPGt_NO_INDICATOR, NULL , 0L, 0L, 0L, ECPGt_EORT, ECPGt_EOLT);
+#line 696 "uvarchar.pgc"
+
+if (sqlca.sqlwarn[0] == 'W') sqlprint();
+#line 696 "uvarchar.pgc"
+
+if (sqlca.sqlcode < 0) sqlprint();}
+#line 696 "uvarchar.pgc"
+
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_var();
+ init_table_value();
+}
+
+void test_all()
+{
+ test_var_1();
+ test_var_2();
+ test_var_3();
+ test_var_4();
+ test_var_5();
+ test_var_6();
+ test_var_7();
+ test_var_8();
+ test_var_9();
+ test_var_10();
+ test_var_11();
+ test_var_12();
+ test_var_13();
+ test_array_1();
+ test_array_2();
+ test_array_3();
+ test_array_4();
+ test_array_5();
+ test_array_6();
+ test_array_7();
+ test_array_8();
+ test_array_9();
+ test_array_10();
+ test_array_11();
+ test_array_12();
+ test_array_13();
+ test_buckinsert_1();
+}
+
+int main()
+{
+ ECPGdebug(1, stderr);
+ if(test_init() !=0)
+ return -1;
+
+ test_all();
+ //test_array_13();
+ //test_var_1();
+ test_finish();
+
+ return 0;
+}
diff --git a/src/interfaces/ecpg/test/unicode/expected/uvarchar.ret b/src/interfaces/ecpg/test/unicode/expected/uvarchar.ret
new file mode 100644
index 0000000..35a03e5
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/expected/uvarchar.ret
@@ -0,0 +1,253 @@
+SQL error:
+
+test_var_1 : simple select into uvarchar var
+---->uvarchar variable,len=15:
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+uvarchar_var_ind = 0
+---->uvarchar variable,len=17:
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+uvarchar_var_ind = 0
+---->uvarchar variable,len=19:
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+uvarchar_var_ind = 0
+---->uvarchar variable,len=20:
+0x4E9A 0x6D32 0x6B27 0x6D32 0x975E 0x6D32 0x5927 0x6D0B
+0x6D32 0x5317 0x7F8E 0x6D32 0x5357 0x7F8E 0x6D32 0x5357
+0x6781 0x6D32 0x6CA1 0x6709
+uvarchar_var_ind = 23
+
+test_var_2 : simple select using uvarchar var
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_3 : simple update using uvarchar
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_var_4 : simple delete using uvarchar var
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_5 : simple insert using uvarchar
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_6 : prepared select into uvarchar var
+---->uvarchar variable,len=15:
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar variable,len=19:
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+
+test_var_7 : prepared select using uvarchar var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_8 : prepared update using uvarchar var
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_9 : prepared delete using uvarchar var
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_10 : prepared insert using uvarchar var
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_11 : Open cursor using uvarchar var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_12 : Fecth cursor into uvarchar var
+---->uvarchar variable,len=15:
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+
+test_var_13 : simple insert using uvarchar with L string inited
+uvarchar_local_var.len = 16
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+---->uvarchar variable,len=15:
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+uvarchar_var_ind = 0
+
+test_var_14 : Open cursor using utext var directly in WHERE Clause
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_15 : Test uvarchar_var working with NULL without indicator
+---->uvarchar variable,len=20:
+0x0000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+found 6 rows for Item being NULL
+
+test_var_16 : Test uvarchar_var working with NULL with indicator
+---->uvarchar variable,len=20:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+uvarchar_var_ind = -1
+found 6 rows for Item being NULL
+
+test_array_1 : simple select into uvarchar array
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=17)
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+---->uvarchar array[2]:(len=19)
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar array[2]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_2 : simple select using uvarchar array
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_3 : simple update using uvarchar array
+find 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+find 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_4 : simple delete using uvarchar array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_5 : simple insert using uvarchar array
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_6 : prepared select into uvarchar array
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=17)
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+---->uvarchar array[2]:(len=19)
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_7 : prepared select using uvarchar array
+count=1 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_8 : prepared update using uvarchar array
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_9 : prepared delete using uvarchar array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_10 : prepared insert using uvarchar array
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_11 : Open cursor using uvarchar array
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_12 : Fecth cursor into uvarchar array
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=17)
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+---->uvarchar array[2]:(len=19)
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_13 : Insert array with L string inited
+---->uvarchar array[0]:(len=15)
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B 0x0000
+0x0000 0x0000 0x0000 0x0000
+---->uvarchar array[1]:(len=17)
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403 0x0000 0x0000 0x0000
+---->uvarchar array[2]:(len=19)
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F 0x0000
+---->uvarchar array[3]:(len=0)
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_15 : Test uvarchar array working with NULL without using indicator
+---->uvarchar variable,len=20:
+0x0000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar variable,len=20:
+0x0000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar variable,len=20:
+0x0000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+found 6 rows for Item being NULL
+
+test_array_16 : Test uvarchar array working with NULL using indicator
+---->uvarchar variable,len=20:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar variable,len=20:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->uvarchar variable,len=20:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+uvarchar_var_ind = -1
+uvarchar_var_ind = -1
+uvarchar_var_ind = -1
+found 6 rows for Item being NULL
diff --git a/src/interfaces/ecpg/test/unicode/readme.txt b/src/interfaces/ecpg/test/unicode/readme.txt
new file mode 100644
index 0000000..9b8570d
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/readme.txt
@@ -0,0 +1 @@
+This test case is used to test the utext/uvarchar on linux platform for communtiy version.
diff --git a/src/interfaces/ecpg/test/unicode/results/utext.ret b/src/interfaces/ecpg/test/unicode/results/utext.ret
new file mode 100644
index 0000000..7e89f6d
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/results/utext.ret
@@ -0,0 +1,435 @@
+SQL error:
+
+test_var_1: simple select into utext var
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E9A 0x00006D32 0x00006B27 0x00006D32 0x0000975E 0x00006D32 0x00005927 0x00006D0B
+0x00006D32 0x00005317 0x00007F8E 0x00006D32 0x00005357 0x00007F8E 0x00006D32 0x00005357
+0x00006781 0x00006D32 0x00006CA1 0x00006709
+======End utext_var content======
+utext_var_ind = 23
+
+test_var_2: simple select using utext var
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_3: simple update using utext var
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_var_4 : simple delete using utext var
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_5 : simple insert using utext
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_6 : prepared select into utext var
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+
+test_var_7 : prepared select using utext var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_8 : prepared update using utext var
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_9 : prepared delete using utext var
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_10 : prepared insert using utext var
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_11 : Open cursor using utext var
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_12 : Fecth cursor into utext var
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+
+test_var_13 : simple insert utext var with L string inited
+======print utext_var_str content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403
+======End utext_var_str content======
+======print utext_var content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+
+test_var_14 : Open cursor using utext var directly in WHERE Clause
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_var_15 : Test utext_var working with NULL without indicator
+======print utext_var content======
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+found 6 rows for Item being NULL
+
+test_var_16 : Test utext_var working with NULL with indicator
+======print utext_var content======
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_array_1 : simple select into utext array
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+---->array[3]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[2]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[3]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_2 : simple select using array
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=3 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_3 : simple update using utext array
+find 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+find 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_4 : simple delete using utext array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 0 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_5 : simple insert using utext array
+found 2 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 2 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_6 : prepared select SQL error:
+into utext array
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+---->array[3]:
+0x00004E9A 0x00006D32 0x00006B27 0x00006D32 0x0000975E 0x00006D32 0x00005927 0x00006D0B
+0x00006D32 0x00005317 0x00007F8E 0x00006D32 0x00005357 0x00007F8E 0x00006D32 0x00005357
+0x00006781 0x00006D32 0x00006CA1 0x00006709
+
+
+test_array_7 : prepared select using utext array
+count=1 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_8 : prepared update using utext array
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_9 : prepared delete using utext array
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_array_10 : prepared insert using utext array
+found 3 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_11 : Open cursor using utext array
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_12 : Fecth cursor into utext array
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+---->array[3]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+
+test_array_13 : Insert array with L string inited
+---->array[0]:
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+---->array[1]:
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x00000000 0x00000000
+---->array[2]:
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00000000 0x00000000 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+
+
+test_array_14 : Open cursor using utext array directly in WHERE Clause
+count=3 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_array_15 : Test utext array working with NULL without using indicator
+---->array[0]:
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[1]:
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[2]:
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+found 6 rows for Item being NULL
+
+test_array_16 : Test utext array working with NULL using indicator
+---->array[0]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[1]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+---->array[2]:
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+
+utext_var_ind = -1
+utext_var_ind = -1
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_pvar_1 : simple select into utext pointer
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403 0x00000000 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+======print utext_var content======
+0x00004E9A 0x00006D32 0x00006B27 0x00006D32 0x0000975E 0x00006D32 0x00005927 0x00006D0B
+0x00006D32 0x00005317 0x00007F8E 0x00006D32 0x00005357 0x00007F8E 0x00006D32 0x00005357
+0x00006781 0x00006D32 0x00006CA1 0x00006709
+======End utext_var content======
+utext_var_ind = 0
+
+test_pvar_2 : simple select using utext pointer
+count=1 for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+count=0 for '世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_3 : simple update using utext pointer
+found 3 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+
+test_pvar_4 : simple delete using utext pointer
+found 0 rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_5 : simple insert using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_6 : prepared select into utext pointer
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+======print utext_var content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F 0x00000000
+======End utext_var content======
+
+test_pvar_7 : prepared select using utext pointer
+count=0 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_8 : prepared update using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_9 : prepared delete using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_10 : prepared insert using utext pointer
+found 1 rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_11 : Open cursor using utext pointer
+count=0 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_12 : Fecth cursor into utext pointer
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+
+test_pvar_13 : simple insert utext pointer with L string inited
+======print utext_var content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B 0x00000000
+0x00000000 0x00000000 0x00000000 0x00000000
+======End utext_var content======
+utext_var_ind = 0
+
+test_pvar_14 : Open cursor using utext var directly in WHERE Clause
+count=0 for Item='世界杯每隔四年就会举行一次每次𠲖个球队'
+
+test_pvar_15 : Test utext pointer working with NULL without using indicator
+======print utext_var content======
+0x00000000 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+found 6 rows for Item being NULL
+
+test_pvar_16 : Test utext pointer working with NULL using indicator
+======print utext_var content======
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161 0x61616161
+0x61616161 0x61616161 0x61616161 0x61616161
+======End utext_var content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_pvar_17 : Test utext uninitialized pointer getting and setting data
+======print utext_var_str content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B
+======End utext_var_str content======
+Find Item = 太𠮷𠜱平洋𠱓大西洋印度洋北冰洋 where Count=18
+
+test_pvar_18 : Test utext uninitialized pointer working with NULL without using indicator
+======print utext_var content======
+0x00000000
+======End utext_var content======
+found 6 rows for Item being NULL
+
+test_pvar_19 : Test utext uninitialized pointer working with NULL using indicator
+======print utext_var content======
+0x00000000
+======End utext_var content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
+
+test_pp_var_1 : Test pointer to pointer utext_var without initialize
+======print utext_var_str content======
+0x0000592A 0x00020BB7 0x00020731 0x00005E73 0x00006D0B 0x00020C53 0x00005927 0x0000897F
+0x00006D0B 0x00005370 0x00005EA6 0x00006D0B 0x00005317 0x000051B0 0x00006D0B
+======End utext_var_str content======
+utext_var_ind = 0
+======print utext_var_str content======
+0x00008DB3 0x00007403 0x00007BEE 0x00007403 0x00007FBD 0x00006BDB 0x00007403 0x00004E52
+0x00004E53 0x00007403 0x00006A44 0x00006984 0x00007403 0x000068D2 0x00007403 0x000051B0
+0x00007403
+======End utext_var_str content======
+utext_var_ind = 0
+======print utext_var_str content======
+0x00004E16 0x0000754C 0x0000676F 0x00006BCF 0x00009694 0x000056DB 0x00005E74 0x00005C31
+0x00004F1A 0x00004E3E 0x0000884C 0x00004E00 0x00006B21 0x00006BCF 0x00006B21 0x00020C96
+0x00004E2A 0x00007403 0x0000961F
+======End utext_var_str content======
+utext_var_ind = 0
+足球篮球羽毛球乒乓球橄榄球棒球冰球
+
+test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator
+======print utext_var_str content======
+
+======End utext_var_str content======
+======print utext_var_str content======
+
+======End utext_var_str content======
+======print utext_var_str content======
+
+======End utext_var_str content======
+found 6 rows for Item being NULL
+
+test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator
+======print utext_var_str content======
+
+======End utext_var_str content======
+utext_var_ind = -1
+======print utext_var_str content======
+
+======End utext_var_str content======
+utext_var_ind = -1
+======print utext_var_str content======
+
+======End utext_var_str content======
+utext_var_ind = -1
+found 6 rows for Item being NULL
diff --git a/src/interfaces/ecpg/test/unicode/results/utext_nor.ret b/src/interfaces/ecpg/test/unicode/results/utext_nor.ret
new file mode 100644
index 0000000..e6ee570
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/results/utext_nor.ret
@@ -0,0 +1 @@
+./utext_nor.sh: line 10: utext: command not found
diff --git a/src/interfaces/ecpg/test/unicode/results/utext_with_no_indicator.ret b/src/interfaces/ecpg/test/unicode/results/utext_with_no_indicator.ret
new file mode 100644
index 0000000..12913ef
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/results/utext_with_no_indicator.ret
@@ -0,0 +1 @@
+./utext.sh: line 12: utext: command not found
diff --git a/src/interfaces/ecpg/test/unicode/unicode.pgc b/src/interfaces/ecpg/test/unicode/unicode.pgc
new file mode 100755
index 0000000..bf9a177
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/unicode.pgc
@@ -0,0 +1,246 @@
+#include <stdio.h>
+
+EXEC SQL INCLUDE ../regression;
+
+void print_utext(char *var_name, utext *utext_var, int var_size);
+void print_utext_array(char *var_name, utext *utext_var, int var_size, int index);
+void print_nchar(char *utext_var, int var_size);
+
+#define print_ret print_utext("employee",employee,40); \
+print_utext("address", address.arr, address.len)
+
+EXEC SQL BEGIN DECLARE SECTION;
+char char_ename[3][40] = {"test ok","测试 通过","太𠮷𠜱"};
+
+char employee_1[40]= "test ok";
+char employee_2[40]= "测试 通过";
+char employee_3[40]= "太𠮷𠜱";
+
+char char_addr[3][40] = {"1 sydney, NSW","澳大利亚悉尼1号","世界杯"};
+
+char address_1[40]="1 sydney, NSW";
+char address_2[40]="澳大利亚悉尼1号";
+char address_3[40]="世界杯";
+EXEC SQL END DECLARE SECTION;
+
+void print_utext(char *var_name, utext *utext_var, int var_size)
+{
+ int i;
+ printf("======print %s content======\n",var_name);
+ for(i=0; i<var_size; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_utext_array(char *var_name, utext *utext_var, int var_size, int index)
+{
+ int i;
+ printf("======print %s[%d] content======\n",var_name,index);
+ for(i=0; i<var_size; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_nchar(char *utext_var, int var_size)
+{
+ printf("nchar is : %s\n",utext_var);
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<var_size; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void create_table()
+{
+ EXEC SQL CREATE TABLE IF NOT EXISTS emp (id int, ename CHAR(20), address nvarchar(50));
+ EXEC SQL TRUNCATE TABLE emp;
+
+ EXEC SQL CREATE TABLE IF NOT EXISTS emp_bk (id int, ename CHAR(20), address nvarchar(50));
+ EXEC SQL TRUNCATE TABLE emp_bk;
+}
+
+void init_table()
+{
+ /* UTF8 is the database character set */
+ EXEC SQL INSERT INTO emp (id,ename,address) VALUES (1, 'test ok', '1 sydney, NSW') ;
+ EXEC SQL INSERT INTO emp (id,ename,address) VALUES (2, '测试 通过', '澳大利亚悉尼1号') ;
+ EXEC SQL INSERT INTO emp (id,ename,address) VALUES (3, '太𠮷𠜱', '世界杯') ;
+}
+
+/*
+ Test utext host variable
+*/
+void testcase1()
+{
+ EXEC SQL BEGIN DECLARE SECTION;
+ int total_tuples = 0;
+ int i = 0;
+ int id = 0;
+ int id2 = 0;
+ utext employee[41]; /* define Unicode host variable */
+ uvarchar address[101] ; /* define a variable length Unicode host variable */
+ EXEC SQL END DECLARE SECTION;
+
+ printf("\n**************Start testcase(%d)**************\n",1);
+
+ memset(employee,'a',sizeof(employee));
+ memset((void*)&address,'a',sizeof(address));
+ address.len = 0;
+
+ EXEC SQL TRUNCATE emp_bk;
+
+ for(i=1;i<=3;i++)
+ {
+ /* Database character set converted to Unicode */
+ EXEC SQL SELECT ename,address INTO :employee, :address FROM emp where id =:i;
+ print_ret;
+ /* Unicode converted to Database character */
+ EXEC SQL INSERT INTO emp_bk (id,ename,address) VALUES (:i,:employee, :address);
+ }
+
+ EXEC SQL SELECT count(*) into :total_tuples from emp_bk;
+ printf("total_tuples = %d in emp_bk\n",total_tuples);
+
+
+ for(i=3;i>0;i--)
+ {
+ EXEC SQL SELECT ename,address INTO :employee, :address FROM emp where id =:i;
+ EXEC SQL SELECT id into :id from emp_bk WHERE ename=:employee and address = :address;
+ EXEC SQL SELECT id into :id2 from emp_bk WHERE ename=:char_ename[i-1] and address=:char_addr[i-1];
+ printf("id[%d] = id2[%d] in emp_bk\n",id,id2);
+ }
+}
+
+/*
+ Test utext array
+*/
+void testcase2()
+{
+ EXEC SQL BEGIN DECLARE SECTION;
+ int total_tuples = 0;
+ int i = 0;
+ int j = 0;
+ int id = 0;
+ int id2 = 0;
+ utext employee_array[3][41]; /* define Unicode host variable */
+ uvarchar address_array[3][101] ; /* define a variable length Unicode host variable */
+ EXEC SQL END DECLARE SECTION;
+
+ printf("\n**************Start testcase(%d)**************\n",2);
+
+ memset(employee_array,'a',sizeof(employee_array));
+
+ for(i=0;i<3;i++)
+ {
+ memset((void*)&address_array[i],'a',sizeof(address_array[0]));
+ address_array[i].len=0;
+ }
+
+
+ EXEC SQL TRUNCATE emp_bk;
+
+ /* Database character set converted to Unicode */
+ EXEC SQL SELECT ename,address INTO :employee_array, :address_array FROM emp;
+
+ for(i=1;i<=3;i++)
+ {
+ print_utext_array("employee_array",employee_array[i-1],40,i-1);
+ print_utext_array("address_array",address_array[i-1].arr,address_array[i-1].len,i-1);
+ /* Unicode converted to Database character */
+ EXEC SQL INSERT INTO emp_bk (id,ename,address) VALUES (:i,:employee_array[i-1], :address_array[i-1]);
+ }
+
+ EXEC SQL SELECT count(*) into :total_tuples from emp_bk;
+ printf("total_tuples = %d in emp_bk\n",total_tuples);
+
+
+ EXEC SQL SELECT ename,address INTO :employee_array, :address_array FROM emp;
+ for(i=3;i>0;i--)
+ {
+ EXEC SQL SELECT id into :id from emp_bk WHERE ename=:employee_array[i-1] and address = :address_array[i-1];
+ EXEC SQL SELECT id into :id2 from emp_bk WHERE ename=:char_ename[i-1] and address=:char_addr[i-1];
+ printf("id[%d] = id2[%d] in emp_bk\n",id,id2);
+ }
+}
+
+
+/*
+ Test utext pointer
+*/
+void testcase3()
+{
+ EXEC SQL BEGIN DECLARE SECTION;
+ int total_tuples = 0;
+ int i = 0;
+ int id = 0;
+ int id2 = 0;
+ utext *employee; /* define Unicode host variable */
+ int employee_len = 82;
+ uvarchar address[101] ; /* define a variable length Unicode host variable */
+ EXEC SQL END DECLARE SECTION;
+
+
+ printf("\n**************Start testcase(%d)**************\n",3);
+
+ employee = malloc(employee_len);
+
+ memset(employee,'a',employee_len);
+ memset((void*)&address,'a',sizeof(address));
+ address.len = 0;
+
+ EXEC SQL TRUNCATE emp_bk;
+
+ for(i=1;i<=3;i++)
+ {
+ /* Database character set converted to Unicode */
+ EXEC SQL SELECT ename,address INTO :employee, :address FROM emp where id =:i;
+ print_ret;
+ /* Unicode converted to Database character */
+ EXEC SQL INSERT INTO emp_bk (id,ename,address) VALUES (:i,:employee, :address);
+ }
+
+ EXEC SQL SELECT count(*) into :total_tuples from emp_bk;
+ printf("total_tuples = %d in emp_bk\n",total_tuples);
+
+
+ for(i=3;i>0;i--)
+ {
+ EXEC SQL SELECT ename,address INTO :employee, :address FROM emp where id =:i;
+ EXEC SQL SELECT id into :id from emp_bk WHERE ename=:employee and address = :address;
+ EXEC SQL SELECT id into :id2 from emp_bk WHERE ename=:char_ename[i-1] and address=:char_addr[i-1];
+ printf("id[%d] = id2[%d] in emp_bk\n",id,id2);
+ }
+}
+
+int main() {
+ EXEC SQL CONNECT TO REGRESSDB1;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+ EXEC SQL WHENEVER SQLWARNING SQLPRINT;
+ EXEC SQL WHENEVER SQLERROR SQLPRINT;
+ create_table();
+ init_table();
+
+ testcase1();
+ testcase2();
+ testcase3();
+
+ EXEC SQL DISCONNECT;
+ return 0;
+}
+
+
+
diff --git a/src/interfaces/ecpg/test/unicode/unicode.sh b/src/interfaces/ecpg/test/unicode/unicode.sh
new file mode 100755
index 0000000..c8eaa87
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/unicode.sh
@@ -0,0 +1,8 @@
+echo "../../preproc/ecpg --regression --enable-utext -I./../../include -I. -o unicode.c unicode.pgc"
+
+../../preproc/ecpg --regression --enable-utext -I./../../include -I. -o unicode.c unicode.pgc
+
+gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -O0 -pthread -D_REENTRANT -D_THREAD_SAFE -D_POSIX_PTHREAD_SEMANTICS -I../../include -I../../../../../src/interfaces/ecpg/include -I../../../../../src/interfaces/libpq -I../../../../../src/include -DLINUX_OOM_SCORE_ADJ=0 -DLINUX_OOM_ADJ=0 -D_GNU_SOURCE -I/db/fep95ae/OUT/libxml2/64/include/libxml2 -I/db/fep95ae/SRC/pclmods/pclmods_rhel7_64/include -I/db/fep95ae/OUT/libedit/64/include -I/db/fep95ae/OUT/OpenLDAP/64/include -I/db/fep95ae/OUT/UUID/64/include -I/db/fep95ae/OUT/libxml2/64/include -I/db/fep95ae/OUT/libxslt/64/include -I/db/fep95ae/OUT/OpenSSL/64/include -I/db/fep95ae/OUT/Kerberos5/64/include -g -c -o unicode.o unicode.c
+gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -O0 -pthread -D_REENTRANT -D_THREAD_SAFE -D_POSIX_PTHREAD_SEMANTICS unicode.o -L../../ecpglib -L../../pgtypeslib -L../../../../../src/interfaces/libpq -L../../../../../src/port -L../../../../../src/common -L/db/fep95ae/OUT/libxml2/64/lib -L/db/fep95ae/SRC/pclmods/pclmods_rhel7_64/lib -L/db/fep95ae/OUT/libedit/64/lib -L/db/fep95ae/OUT/OpenLDAP/64/lib -L/db/fep95ae/OUT/UUID/64/lib -L/db/fep95ae/OUT/libxml2/64/lib -L/db/fep95ae/OUT/libxslt/64/lib -L/db/fep95ae/OUT/OpenSSL/64/lib -L/db/fep95ae/OUT/Kerberos5/64/lib -Wl,--as-needed -Wl,-rpath,'/db/fep95ae/TMP/postgres/lib',--enable-new-dtags -lecpg -lpgtypes -lpq -lpgcommon -lpgport -lselinux -lxslt -lxml2 -lpam -lssl -lcrypto -lgssapi_krb5 -lz -ledit -lrt -lcrypt -ldl -lm -g -o unicode
+
+
diff --git a/src/interfaces/ecpg/test/unicode/utext.pgc b/src/interfaces/ecpg/test/unicode/utext.pgc
new file mode 100755
index 0000000..411de30
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/utext.pgc
@@ -0,0 +1,1477 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <wchar.h>
+EXEC SQL INCLUDE ../regression;
+
+#define test(msg) printf("\n%s\n",msg)
+#define VAR_SIZE 20
+#define ARRAY_SIZE 4
+#define P_VAR_SIZE 22
+
+
+/* Following is UTF8 and UTF16/UTF32 characters mapping table */
+/*太𠮷𠜱平洋𠱓大西洋印度洋北冰洋
+utf16
+0x592A 0xD842 0xDFB7 0xD841 0xDF31 0x5E73 0x6D0B 0xD843
+0xDC53 0x5927 0x897F 0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317
+0x51B0 0x6D0B 0x0000,0x0000
+
+utf32
+0x592A 0x20BB7 0x20731 0x5E73 0x6D0B 0x20C53 0x5927 0x897F
+0x6D0B 0x5370 0x5EA6 0x6D0B 0x5317 0x51B0 0x6D0B
+*/
+
+/*足球篮球羽毛球乒乓球橄榄球棒球冰球
+utf16
+0x8DB3,0x7403,0x7BEE,0x7403,0x7FBD,0x6BDB,0x7403,0x4E52,
+0x4E53,0x7403,0x6A44,0x6984,0x7403,0x68D2,0x7403,0x51B0,
+0x7403,0x0000,0x0000,0x0000
+
+utf32
+0x8DB3 0x7403 0x7BEE 0x7403 0x7FBD 0x6BDB 0x7403 0x4E52
+0x4E53 0x7403 0x6A44 0x6984 0x7403 0x68D2 0x7403 0x51B0
+0x7403
+*/
+
+/*世界杯每隔四年就会举行一次每次𠲖个球队
+utf16
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0xD843
+0xDC96 0x4E2A 0x7403 0x961F
+
+utf32
+0x4E16 0x754C 0x676F 0x6BCF 0x9694 0x56DB 0x5E74 0x5C31
+0x4F1A 0x4E3E 0x884C 0x4E00 0x6B21 0x6BCF 0x6B21 0x20C96
+0x4E2A 0x7403 0x961F
+*/
+
+/* 亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲
+0x4E9A,0x6D32,0x6B27,0x6D32,0x975E,0x6D32,0x5927,0x6D0B,
+0x6D32,0x5317,0x7F8E,0x6D32,0x5357,0x7F8E,0x6D32,0x5357,
+0x6781,0x6D32,0x6CA1,0x6709,0x5317,0x6781,0x6D32
+*/
+/* 太𠮷
+0x592A 0xD842 0xDFB7
+*/
+void init_var(void);
+void test_var_1(void);
+void test_var_2(void);
+void test_var_3(void);
+void test_var_4(void);
+void test_var_5(void);
+void test_var_6(void);
+void test_var_7(void);
+void test_var_8(void);
+void test_var_9(void);
+void test_var_10(void);
+void test_var_11(void);
+void test_var_12(void);
+void test_var_13(void);
+void test_var_14(void);
+void test_var_15(void);
+void test_var_16(void);
+void test_array_1(void);
+void test_array_2(void);
+void test_array_3(void);
+void test_array_4(void);
+void test_array_5(void);
+void test_array_6(void);
+void test_array_7(void);
+void test_array_8(void);
+void test_array_9(void);
+void test_array_10(void);
+void test_array_11(void);
+void test_array_12(void);
+void test_array_13(void);
+void test_array_14(void);
+void test_array_15(void);
+void test_array_16(void);
+void test_pvar_1(void);
+void test_pvar_2(void);
+void test_pvar_3(void);
+void test_pvar_4(void);
+void test_pvar_5(void);
+void test_pvar_6(void);
+void test_pvar_7(void);
+void test_pvar_8(void);
+void test_pvar_9(void);
+void test_pvar_10(void);
+void test_pvar_11(void);
+void test_pvar_12(void);
+void test_pvar_13(void);
+void test_pvar_14(void);
+void test_pvar_15(void);
+void test_pvar_16(void);
+void test_pvar_17(void);
+void test_pvar_18(void);
+void test_pvar_19(void);
+void test_pp_var_1(void);
+void test_pp_var_2(void);
+void test_pp_var_3(void);
+void test_desc_1(void);
+void test_all(void);
+
+void print_utext(utext *utext_var);
+void print_utext_str(utext *utext_var);
+void print_utext_ind(int utext_var_ind);
+void print_utext_size(utext *utext_var,int size);
+void print_array(void *array, int array_size, int var_size);
+void print_array_with_index(int index);
+int test_init(void);
+void test_finish(void);
+void init_table_value(void);
+
+EXEC SQL BEGIN DECLARE SECTION;
+ int utext_var_size = 20;
+ int utext_array_size = 4;
+ int count=0;
+ int total_tuples = 0;
+ int i;
+ char char_var[10]={0xF0,0x90,0x90,0xB7};
+
+ utext utext_var[VAR_SIZE];
+ utext utext_array[ARRAY_SIZE][VAR_SIZE];
+ utext utext_input_var[20]={0x592a,0xd842, 0xdfb7, 0x0000};
+ utext *p_utext_var = NULL;
+
+
+ int utext_var_ind;
+ int count_array[4]={1,2,3,4};
+
+EXEC SQL END DECLARE SECTION;
+
+void print_utext(utext *utext_var)
+{
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<VAR_SIZE; i++)
+ {
+ printf ("0x%08X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void print_utext_str(utext *utext_var)
+{
+ int i=0;
+ printf("======print utext_var_str content======\n");
+ if (utext_var==0)
+ {
+ printf("utext_var is NULL\n");
+ printf("\n======End utext_var_str content======\n");
+ return;
+ }
+ while(utext_var[i] != 0)
+ {
+ printf ("0x%08X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ i++;
+ if(i>100)
+ break;
+ }
+ printf("\n======End utext_var_str content======\n");
+}
+
+void print_utext_size(utext *utext_var,int size)
+{
+ int i;
+ printf("======print utext_var content======\n");
+ for(i=0; i<size; i++)
+ {
+ printf ("0x%08X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n======End utext_var content======\n");
+}
+
+void print_utext_ind(int utext_var_ind)
+{
+ printf("utext_var_ind = %d\n",utext_var_ind);
+}
+void print_array(void *array, int array_size, int var_size)
+{
+ int i,j;
+
+ for(i=0; i<array_size; i++)
+ {
+ utext *utext_array = &((utext*)array)[i*var_size];
+
+ printf ("---->array[%d]:\n", i);
+
+ for(j=0; j<var_size; j++)
+ {
+ printf ("0x%08X ", utext_array[j]);
+ if(j>6 && (j+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+ }
+
+ printf("\n");
+}
+
+int test_init()
+{
+ p_utext_var = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ if(!p_utext_var)
+ {
+ printf("Error: failed allco memory for p_utext_var. \n");
+ return -1;
+ }
+
+ EXEC SQL CONNECT TO REGRESSDB1;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+ EXEC SQL WHENEVER SQLWARNING SQLPRINT;
+ EXEC SQL WHENEVER SQLERROR SQLPRINT;
+
+ EXEC SQL SET client_encoding='UTF8';
+
+ //initialization of test table
+ EXEC SQL CREATE TABLE IF NOT EXISTS tb1 (Item varchar, Count integer);
+
+ init_table_value();
+
+ return 0;
+}
+
+void test_finish()
+{
+ EXEC SQL DROP TABLE tb1;
+ EXEC SQL DISCONNECT ALL;
+}
+
+void init_table_value()
+{
+ EXEC SQL TRUNCATE tb1;
+
+ EXEC SQL INSERT INTO tb1 VALUES ('太𠮷𠜱平洋𠱓大西洋印度洋北冰洋', 1);
+ EXEC SQL INSERT INTO tb1 VALUES ('足球篮球羽毛球乒乓球橄榄球棒球冰球', 2);
+ EXEC SQL INSERT INTO tb1 VALUES ('世界杯每隔四年就会举行一次每次𠲖个球队', 3);
+ EXEC SQL INSERT INTO tb1 VALUES ('亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲', 4);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(8);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(9);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(10);
+}
+
+void init_var()
+{
+ count = 0;
+ memset(utext_var,'a',sizeof(utext_var));
+ memset(utext_array,'a',sizeof(utext_array));
+ memset(p_utext_var,'a',P_VAR_SIZE*sizeof(utext));
+}
+
+//simple select into utext var
+void test_var_1()
+{
+ test("test_var_1: simple select into utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=1;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=2;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=3;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=4;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+}
+
+
+//simple select using utext var
+void test_var_2()
+{
+ test("test_var_2: simple select using utext var");
+ init_var();
+
+ count = 0;
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:utext_var;
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:utext_var;
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using utext var
+void test_var_3()
+{
+ test("test_var_3: simple update using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL UPDATE tb1 SET Item=:utext_var WHERE Count=2;
+ EXEC SQL UPDATE tb1 SET Item=:utext_var WHERE Count=3;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext var
+void test_var_4()
+{
+ test("test_var_4 : simple delete using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext var
+void test_var_5()
+{
+ test("test_var_5 : simple insert using utext");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL INSERT INTO tb1 values (:utext_var, 11);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL INSERT INTO tb1 values (:utext_var, 13);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext var
+void test_var_6()
+{
+ test("test_var_6 : prepared select into utext var");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=?";
+
+ EXEC SQL EXECUTE stmt INTO :utext_var USING 1;
+ print_utext(utext_var);
+
+ EXEC SQL EXECUTE stmt INTO :utext_var USING 3;
+ print_utext(utext_var);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared select using utext var
+void test_var_7()
+{
+ test("test_var_7 : prepared select using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :utext_var;
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using utext var
+void test_var_8()
+{
+ test("test_var_8 : prepared update using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :utext_var, 1;
+ EXEC SQL EXECUTE stmt USING :utext_var, 2;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using utext var
+void test_var_9()
+{
+ test("test_var_9 : prepared delete using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using utext var
+void test_var_10()
+{
+ test("test_var_10 : prepared insert using utext var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, 13)";
+ EXEC SQL EXECUTE stmt USING :utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using utext var
+void test_var_11()
+{
+ test("test_var_11 : Open cursor using utext var");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_var_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_var_11 USING :utext_var;
+ EXEC SQL FETCH cursor_var_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_var_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into utext var
+void test_var_12()
+{
+ test("test_var_12 : Fecth cursor into utext var");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=1";
+ EXEC SQL DECLARE cursor_var_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_var_12;
+ EXEC SQL FETCH cursor_var_12 INTO :utext_var;
+ EXEC SQL CLOSE cursor_var_12;
+
+ print_utext(utext_var);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//simple insert utext var with L string inited
+void test_var_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[]=L"足球篮球羽毛球";
+EXEC SQL END DECLARE SECTION;
+ test("test_var_13 : simple insert utext var with L string inited");
+ init_var();
+
+ print_utext_str(utext_local_var);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var, 16);
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=16;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+ init_table_value();
+}
+
+//Open cursor using utext var directly in WHERE Clause
+void test_var_14()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[20+1];
+EXEC SQL END DECLARE SECTION;
+ memset(utext_local_var,'a',sizeof(utext_local_var));
+
+ test("test_var_14 : Open cursor using utext var directly in WHERE Clause");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_local_var FROM tb1 WHERE Count=3;
+ EXEC SQL DECLARE cursor_var_14 CURSOR FOR SELECT Count FROM tb1 WHERE Item=:utext_local_var;
+ EXEC SQL OPEN cursor_var_14;
+ EXEC SQL FETCH cursor_var_14 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ EXEC SQL CLOSE cursor_var_14;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Test utext_var working with NULL without indicator
+void test_var_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[20+1];
+EXEC SQL END DECLARE SECTION;
+ test("test_var_15 : Test utext_var working with NULL without indicator");
+
+ memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_var FROM tb1 WHERE Count=8;
+ print_utext(utext_local_var);
+
+ memset(utext_local_var,0x00,sizeof(utext_local_var));
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext_var working with NULL with indicator
+void test_var_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[20+1];
+ int utext_var_ind=-1;
+EXEC SQL END DECLARE SECTION;
+ test("test_var_16 : Test utext_var working with NULL with indicator");
+
+ memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_var:utext_var_ind FROM tb1 WHERE Count=8;
+ print_utext(utext_local_var);
+ print_utext_ind(utext_var_ind);
+
+ memset(utext_local_var,0x00,sizeof(utext_local_var));
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var:utext_var_ind, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var:utext_var_ind, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_var:utext_var_ind, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+
+//simple select into utext array
+void test_array_1()
+{
+ test("test_array_1 : simple select into utext array ");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array[0] FROM tb1 WHERE Count=1;
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+ init_var();
+}
+
+//simple select using utext array
+void test_array_2()
+{
+ test("test_array_2 : simple select using array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:utext_array[0];
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:utext_array[2];
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using utext array
+void test_array_3()
+{
+ test("test_array_3 : simple update using utext array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL UPDATE tb1 SET Item=:utext_array[2] WHERE Count=1;
+
+ count = 0;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("find %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("find %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext array
+void test_array_4()
+{
+ test("test_array_4 : simple delete using utext array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ count = 100;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:utext_array[0];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 100;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:utext_array[2];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext array
+void test_array_5()
+{
+ test("test_array_5 : simple insert using utext array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL INSERT INTO tb1 values (:utext_array[0], 11);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_array[2], 13);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext array
+void test_array_6()
+{
+ test("test_array_6 : prepared select into utext array");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count<=?";
+
+ EXEC SQL EXECUTE stmt INTO :utext_array USING 4;
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared select using utext array
+void test_array_7()
+{
+ test("test_array_7 : prepared select using utext array");
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :utext_array[0];
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using utext array
+void test_array_8()
+{
+ test("test_array_8 : prepared update using utext array");
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :utext_array[0], 2;
+ EXEC SQL EXECUTE stmt USING :utext_array[0], 3;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using utext array
+void test_array_9()
+{
+ test("test_array_9 : prepared delete using utext array");
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :utext_array[0];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using utext array
+void test_array_10()
+{
+ test("test_array_10 : prepared insert using utext array");
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, ?)";
+ EXEC SQL EXECUTE stmt USING :utext_array[2], 13;
+ EXEC SQL EXECUTE stmt USING :utext_array[2], 15;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using utext array
+void test_array_11()
+{
+ test("test_array_11 : Open cursor using utext array");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_array_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_array_11 USING :utext_array[2];
+ EXEC SQL FETCH cursor_array_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_array_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into utext array
+void test_array_12()
+{
+ test("test_array_12 : Fecth cursor into utext array");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count<=3";
+ EXEC SQL DECLARE cursor_array_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_array_12;
+ EXEC SQL FETCH cursor_array_12 INTO :utext_array[0];
+ EXEC SQL FETCH cursor_array_12 INTO :utext_array[1];
+ EXEC SQL FETCH cursor_array_12 INTO :utext_array[2];
+ EXEC SQL CLOSE cursor_array_12;
+
+ print_array(utext_array,ARRAY_SIZE,VAR_SIZE);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Insert array with L string inited
+void test_array_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_array13[4][VAR_SIZE]={L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋",L"足球篮球羽毛球乒乓球橄榄球棒球冰球",L"世界杯每隔四年就会举行一次"};
+EXEC SQL END DECLARE SECTION;
+
+ test("test_array_13 : Insert array with L string inited");
+ init_var();
+
+
+ EXEC SQL INSERT INTO tb1 values (:utext_array13[0], 20);
+ EXEC SQL INSERT INTO tb1 values (:utext_array13[1], 21);
+ EXEC SQL INSERT INTO tb1 values (:utext_array13[2], 22);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :utext_array FROM tb1 WHERE Count<=22 AND Count>=20;
+ print_array(utext_array,3,VAR_SIZE);
+ init_table_value();
+}
+
+//Open cursor using utext array directly in WHERE Clause
+void test_array_14()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_array[4][20+1];
+EXEC SQL END DECLARE SECTION;
+
+ test("test_array_14 : Open cursor using utext array directly in WHERE Clause");
+ memset(utext_local_array,'a',sizeof(utext_local_array));
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_local_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL DECLARE cursor_array_14 CURSOR FOR SELECT Count FROM tb1 WHERE Item=:utext_local_array[2];
+ EXEC SQL OPEN cursor_array_14;
+ EXEC SQL FETCH cursor_array_14 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_array_14;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+
+//utext array working with NULL without using indicator
+void test_array_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_array[3][20];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_15 : Test utext array working with NULL without using indicator");
+ memset(utext_local_array,'a',sizeof(utext_local_array));
+
+ EXEC SQL SELECT Item INTO :utext_local_array FROM tb1 WHERE Count>=8 and Count<=10;
+
+ print_array((void**)utext_local_array,3,20);
+
+ memset(utext_local_array,0x00,sizeof(utext_local_array));
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext array working with NULL using indicator
+void test_array_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_array[3][20];
+ int utext_array_ind[3];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_16 : Test utext array working with NULL using indicator");
+ memset(utext_local_array,'a',sizeof(utext_local_array));
+
+ EXEC SQL SELECT Item INTO :utext_local_array:utext_array_ind FROM tb1 WHERE Count>=8 and Count<=10;
+
+ print_array((void**)utext_local_array,3,20);
+ print_utext_ind(utext_array_ind[0]);
+ print_utext_ind(utext_array_ind[1]);
+ print_utext_ind(utext_array_ind[2]);
+
+ memset(utext_local_array,0x00,sizeof(utext_local_array));
+ utext_array_ind[0]=-1;
+ utext_array_ind[1]=-1;
+ utext_array_ind[2]=-1;
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[0]:utext_array_ind[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[1]:utext_array_ind[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_array[2]:utext_array_ind[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+
+//simple select into utext pointer
+void test_pvar_1()
+{
+ test("test_pvar_1 : simple select into utext pointer");
+ EXEC SQL SELECT Item INTO :p_utext_var:utext_var_ind FROM tb1 WHERE Count=1;
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var:utext_var_ind FROM tb1 WHERE Count=2;
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var:utext_var_ind FROM tb1 WHERE Count=3;
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var:utext_var_ind FROM tb1 WHERE Count=4;
+ print_utext(p_utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+
+ init_table_value();
+}
+
+
+//simple select using utext pointer
+void test_pvar_2()
+{
+ test("test_pvar_2 : simple select using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:p_utext_var;
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:p_utext_var;
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+
+//simple update using utext pointer
+void test_pvar_3()
+{
+ test("test_pvar_3 : simple update using utext pointer");
+ init_var();
+
+ count = 0;
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL UPDATE tb1 SET Item=:p_utext_var WHERE Count=2;
+ EXEC SQL UPDATE tb1 SET Item=:p_utext_var WHERE Count=3;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using utext pointer
+void test_pvar_4()
+{
+ test("test_pvar_4 : simple delete using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=1;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:p_utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:p_utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using utext pointer
+void test_pvar_5()
+{
+ test("test_pvar_5 : simple insert using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+ EXEC SQL INSERT INTO tb1 values (:p_utext_var, 13);
+
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into utext pointer
+void test_pvar_6()
+{
+ test("test_pvar_6 : prepared select into utext pointer");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=?";
+
+ EXEC SQL EXECUTE stmt INTO :p_utext_var USING 1;
+ print_utext(p_utext_var);
+
+ init_var();
+ EXEC SQL EXECUTE stmt INTO :p_utext_var USING 3;
+ print_utext(p_utext_var);
+ init_table_value();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+}
+
+//prepared select using utext pointer
+void test_pvar_7()
+{
+ test("test_pvar_7 : prepared select using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :p_utext_var;
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using utext pointer
+void test_pvar_8()
+{
+ test("test_pvar_8 : prepared update using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :p_utext_var, 1;
+ EXEC SQL EXECUTE stmt USING :p_utext_var, 2;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using utext pointer
+void test_pvar_9()
+{
+ test("test_pvar_9 : prepared delete using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :p_utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using utext pointer
+void test_pvar_10()
+{
+ test("test_pvar_10 : prepared insert using utext pointer");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, 13)";
+ EXEC SQL EXECUTE stmt USING :p_utext_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using utext pointer
+void test_pvar_11()
+{
+ test("test_pvar_11 : Open cursor using utext pointer");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :p_utext_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_pvar_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_pvar_11 USING :p_utext_var;
+ EXEC SQL FETCH cursor_pvar_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_pvar_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into utext pointer
+void test_pvar_12()
+{
+ test("test_pvar_12 : Fecth cursor into utext pointer");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=1";
+ EXEC SQL DECLARE cursor_pvar_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_pvar_12;
+ EXEC SQL FETCH cursor_pvar_12 INTO :p_utext_var;
+ EXEC SQL CLOSE cursor_pvar_12;
+
+ print_utext(p_utext_var);
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//simple insert utext pointer with L string inited
+void test_pvar_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar=L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋";
+EXEC SQL END DECLARE SECTION;
+ test("test_pvar_13 : simple insert utext pointer with L string inited");
+ init_var();
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 16);
+
+ EXEC SQL SELECT Item INTO :utext_var:utext_var_ind FROM tb1 WHERE Count=16;
+ print_utext(utext_var);
+ print_utext_ind(utext_var_ind);
+ init_var();
+ init_table_value();
+}
+
+//Open cursor using utext var directly in WHERE Clause
+void test_pvar_14()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar;
+EXEC SQL END DECLARE SECTION;
+ utext_local_pvar = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ memset(utext_local_pvar,'a',P_VAR_SIZE*sizeof(utext));
+
+ test("test_pvar_14 : Open cursor using utext var directly in WHERE Clause");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :utext_local_pvar FROM tb1 WHERE Count=3;
+
+ EXEC SQL DECLARE cursor_pvar_14 CURSOR FOR SELECT Count FROM tb1 WHERE Item=:utext_local_pvar;
+ EXEC SQL OPEN cursor_pvar_14;
+ EXEC SQL FETCH cursor_pvar_14 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_pvar_14;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+
+//utext pointer working with NULL without using indicator
+void test_pvar_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar;
+EXEC SQL END DECLARE SECTION;
+ utext_local_pvar = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ memset(utext_local_pvar,'a',P_VAR_SIZE*sizeof(utext));
+
+ test("test_pvar_15 : Test utext pointer working with NULL without using indicator");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar FROM tb1 WHERE Count=8;
+
+ print_utext(utext_local_pvar);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext pointer working with NULL using indicator
+void test_pvar_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar;
+ int utext_pvar_ind = -1;
+EXEC SQL END DECLARE SECTION;
+ utext_local_pvar = (utext*)malloc(P_VAR_SIZE*sizeof(utext));
+ memset(utext_local_pvar,'a',P_VAR_SIZE*sizeof(utext));
+
+ test("test_pvar_16 : Test utext pointer working with NULL using indicator");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar:utext_pvar_ind FROM tb1 WHERE Count=8;
+
+ print_utext(utext_local_pvar);
+ print_utext_ind(utext_pvar_ind);
+
+ memset(utext_local_pvar,0,P_VAR_SIZE*sizeof(utext));
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext uninitialized pointer getting and setting data
+void test_pvar_17()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar=0;
+ char local_str[50];
+EXEC SQL END DECLARE SECTION;
+
+ test("test_pvar_17 : Test utext uninitialized pointer getting and setting data");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar FROM tb1 WHERE Count=1;
+
+ print_utext_str(utext_local_pvar);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 18);
+ EXEC SQL SELECT Item INTO :local_str FROM tb1 WHERE Count=18;
+ printf ("Find Item = %s where Count=18\n", local_str);
+
+ init_table_value();
+}
+
+//utext uninitialized pointer working with NULL without using indicator
+void test_pvar_18()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar=0;
+EXEC SQL END DECLARE SECTION;
+
+ test("test_pvar_18 : Test utext uninitialized pointer working with NULL without using indicator");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar FROM tb1 WHERE Count=8;
+
+ print_utext_size(utext_local_pvar,1);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//utext uninitialized pointer working with NULL using indicator
+void test_pvar_19()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext *utext_local_pvar=0;
+ int utext_pvar_ind = -1;
+EXEC SQL END DECLARE SECTION;
+
+ test("test_pvar_19 : Test utext uninitialized pointer working with NULL using indicator");
+
+ EXEC SQL SELECT Item INTO :utext_local_pvar:utext_pvar_ind FROM tb1 WHERE Count=8;
+
+ print_utext_size(utext_local_pvar,1);
+ print_utext_ind(utext_pvar_ind);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_pvar:utext_pvar_ind, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//Test select into host varaibles from descriptor
+void test_desc_1()
+{
+ EXEC SQL BEGIN DECLARE SECTION;
+ utext utext_local_var[20];
+ utext *utext_local_pvar=0;
+ utext **utext_local_ppvar=0;
+ char desc1[8] = "outdesc";
+ EXEC SQL END DECLARE SECTION;
+
+// ECPGdebug(1, stderr);
+
+ EXEC SQL ALLOCATE DESCRIPTOR indesc;
+ EXEC SQL ALLOCATE DESCRIPTOR :desc1;
+
+ EXEC SQL SELECT Item,Count INTO SQL DESCRIPTOR :desc1 FROM tb1 WHERE Count=1;
+
+ EXEC SQL GET DESCRIPTOR :desc1 VALUE 1 :utext_local_var = DATA;
+ print_utext(utext_local_var);
+
+ EXEC SQL GET DESCRIPTOR :desc1 VALUE 1 :utext_local_pvar = DATA;
+ print_utext_str(utext_local_pvar);
+
+
+ EXEC SQL SELECT Item,Count INTO SQL DESCRIPTOR :desc1 FROM tb1 WHERE Count<=3;
+ EXEC SQL GET DESCRIPTOR :desc1 VALUE 1 :utext_local_ppvar = DATA;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_str(utext_local_ppvar[2]);
+
+
+ EXEC SQL SELECT Item,Count INTO SQL DESCRIPTOR :desc1 FROM tb1 WHERE Count<=10 and Count>=8;
+ EXEC SQL GET DESCRIPTOR :desc1 VALUE 1 :utext_local_ppvar = DATA;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_str(utext_local_ppvar[2]);
+
+
+ EXEC SQL DEALLOCATE DESCRIPTOR indesc;
+ EXEC SQL DEALLOCATE DESCRIPTOR :desc1;
+}
+
+
+//Test uninitialized pointer to pointer utext_var without initialize
+void test_pp_var_1()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext **utext_local_ppvar=0;
+ int *utext_local_ppvar_ind=0;
+ char utext_local_char[80];
+EXEC SQL END DECLARE SECTION;
+ test("test_pp_var_1 : Test pointer to pointer utext_var without initialize");
+
+ //memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_ppvar:utext_local_ppvar_ind FROM tb1 WHERE Count<=3;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_ind(utext_local_ppvar_ind[0]);
+
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_ind(utext_local_ppvar_ind[1]);
+
+ print_utext_str(utext_local_ppvar[2]);
+ print_utext_ind(utext_local_ppvar_ind[2]);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[1], 28);
+ EXEC SQL SELECT Item INTO :utext_local_char FROM tb1 WHERE Count=28;
+ printf("%s\n",utext_local_char);
+
+ init_table_value();
+}
+
+
+//Test uninitialized pointer to pointer utext_var working with NULL without indicator
+void test_pp_var_2()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext **utext_local_ppvar=0;
+EXEC SQL END DECLARE SECTION;
+ test("test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator");
+
+ //memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_ppvar FROM tb1 WHERE Count>=8 and Count<=10;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_str(utext_local_ppvar[2]);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//Test uninitialized pointer to pointer utext_var working with NULL with indicator
+void test_pp_var_3()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ utext **utext_local_ppvar=0;
+ int *utext_local_ppvar_ind=0;
+EXEC SQL END DECLARE SECTION;
+ test("test_pp_var_2 : Test uninitialized pointer to pointer utext_var working with NULL without indicator");
+
+ //memset(utext_local_var,'a',sizeof(utext_local_var));
+ EXEC SQL SELECT Item INTO :utext_local_ppvar:utext_local_ppvar_ind FROM tb1 WHERE Count>=8 and Count<=10;
+ print_utext_str(utext_local_ppvar[0]);
+ print_utext_ind(utext_local_ppvar_ind[0]);
+
+ print_utext_str(utext_local_ppvar[1]);
+ print_utext_ind(utext_local_ppvar_ind[1]);
+
+ print_utext_str(utext_local_ppvar[2]);
+ print_utext_ind(utext_local_ppvar_ind[2]);
+
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:utext_local_ppvar[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+void test_all()
+{
+ test_var_1();
+ test_var_2();
+ test_var_3();
+ test_var_4();
+ test_var_5();
+ test_var_6();
+ test_var_7();
+ test_var_8();
+ test_var_9();
+ test_var_10();
+ test_var_11();
+ test_var_12();
+ test_var_13();
+ test_var_14();
+ test_var_15();
+ test_var_16();
+ test_array_1();
+ test_array_2();
+ test_array_3();
+ test_array_4();
+ test_array_5();
+ test_array_6();
+ test_array_7();
+ test_array_8();
+ test_array_9();
+ test_array_10();
+ test_array_11();
+ test_array_12();
+ test_array_13();
+ test_array_14();
+ test_array_15();
+ test_array_16();
+ test_pvar_1();
+ test_pvar_2();
+ test_pvar_3();
+ test_pvar_4();
+ test_pvar_5();
+ test_pvar_6();
+ test_pvar_7();
+ test_pvar_8();
+ test_pvar_9();
+ test_pvar_10();
+ test_pvar_11();
+ test_pvar_12();
+ test_pvar_13();
+ test_pvar_14();
+ test_pvar_15();
+ test_pvar_16();
+ test_pvar_17();
+ test_pvar_18();
+ test_pvar_19();
+ test_pp_var_1();
+ test_pp_var_2();
+ test_pp_var_3();
+}
+
+int main(int argc, char *argv[])
+{
+// ECPGdebug(1, stderr);
+ if(test_init() !=0)
+ return -1;
+
+ test_all();
+ test_finish();
+
+ return 0;
+}
diff --git a/src/interfaces/ecpg/test/unicode/utext.sh b/src/interfaces/ecpg/test/unicode/utext.sh
new file mode 100755
index 0000000..0ee49b2
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/utext.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/bash
+
+echo "../../preproc/ecpg --regression --enable-utext -I./../../include -I. -o utext.c utext.pgc"
+
+#../../preproc/ecpg --regression --enable-utext -I./../../include -I. -o utext.c utext.pgc
+
+
+../../preproc/ecpg --regression -r no_indicator --enable-utext -I./../../include -I. -o utext.c utext.pgc
+
+gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -O0 -pthread -D_REENTRANT -D_THREAD_SAFE -D_POSIX_PTHREAD_SEMANTICS -I../../include -I../../../../../src/interfaces/ecpg/include -I../../../../../src/interfaces/libpq -I../../../../../src/include -DLINUX_OOM_SCORE_ADJ=0 -DLINUX_OOM_ADJ=0 -D_GNU_SOURCE -I/db/pgmaster/run/include -g -c -o utext.o utext.c
+
+
+gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -O0 -pthread -D_REENTRANT -D_THREAD_SAFE -D_POSIX_PTHREAD_SEMANTICS utext.o -L../../ecpglib -L../../pgtypeslib -L../../../../../src/interfaces/libpq -L../../../../../src/port -L../../../../../src/common -L/db/pgmaster/run/lib -Wl,--as-needed -Wl,-rpath,--enable-new-dtags -lecpg -lpgtypes -lpq -lpgcommon -lpgport -lselinux -lxslt -lxml2 -lpam -lssl -lcrypto -lgssapi_krb5 -lz -lrt -lcrypt -ldl -lm -g -o utext
+
+
+./utext >results/utext.ret 2>&1
diff --git a/src/interfaces/ecpg/test/unicode/uvarchar.pgc b/src/interfaces/ecpg/test/unicode/uvarchar.pgc
new file mode 100755
index 0000000..4dd894a
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/uvarchar.pgc
@@ -0,0 +1,877 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+EXEC SQL INCLUDE ../regression;
+
+#define test(msg) printf("\n%s\n",msg)
+#define VAR_SIZE 20
+#define ARRAY_SIZE 4
+
+/* Following is UTF8 and UTF16 characters mapping table */
+/*太𠮷𠜱平洋𠱓大西洋印度洋北冰洋
+0x592A,0x5E73,0x6D0B,0x5927,0x897F,0x6D0B,0x5370,0x5EA6,
+0x6D0B,0x5317,0x51B0,0x6D0B,0x548C,0x5357,0x51B0,0x6D0B,
+0x0000,0x0000,0x0000,0x0000*/
+
+/*足球篮球羽毛球乒乓球橄榄球棒球冰球
+0x8DB3,0x7403,0x7BEE,0x7403,0x7FBD,0x6BDB,0x7403,0x4E52,
+0x4E53,0x7403,0x6A44,0x6984,0x7403,0x68D2,0x7403,0x51B0,
+0x7403,0x0000,0x0000,0x0000
+*/
+
+/*世界杯每隔四年就会举行一次每次𠲖个球队
+0x4E16,0x754C,0x676F,0x6BCF,0x9694,0x56DB,0x5E74,0x5C31,
+0x4F1A,0x4E3E,0x884C,0x4E00,0x6B21,0x6BCF,0x6B21,0x0033,
+0x0032,0x4E2A,0x7403,0x961F
+*/
+
+/* 亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲
+0x4E9A,0x6D32,0x6B27,0x6D32,0x975E,0x6D32,0x5927,0x6D0B,
+0x6D32,0x5317,0x7F8E,0x6D32,0x5357,0x7F8E,0x6D32,0x5357,
+0x6781,0x6D32,0x6CA1,0x6709,0x5317,0x6781,0x6D32
+*/
+
+EXEC SQL BEGIN DECLARE SECTION;
+ UVARCHAR uvarchar_var[VAR_SIZE];
+ UVARCHAR uvarchar_array[ARRAY_SIZE][VAR_SIZE];
+
+ int uvarchar_var_ind;
+
+ int count;
+ int count_array[4]={1,2,3,4};
+ int total_tuples = 0;
+EXEC SQL END DECLARE SECTION;
+
+void print_uvarchar(void);
+void print_uvarchar_ind(int uvarchar_var_ind);
+void print_local_uvarchar(utext *utext_var, int var_len);
+void print_array(void);
+int test_init(void);
+void test_finish(void);
+void init_table_value(void);
+void init_var(void);
+
+void test_var_1(void);
+void test_var_2(void);
+void test_var_3(void);
+void test_var_4(void);
+void test_var_5(void);
+void test_var_6(void);
+void test_var_7(void);
+void test_var_8(void);
+void test_var_9(void);
+void test_var_10(void);
+void test_var_11(void);
+void test_var_12(void);
+void test_var_13(void);
+void test_var_14(void);
+void test_var_15(void);
+void test_var_16(void);
+void test_array_1(void);
+void test_array_2(void);
+void test_array_3(void);
+void test_array_4(void);
+void test_array_5(void);
+void test_array_6(void);
+void test_array_7(void);
+void test_array_8(void);
+void test_array_9(void);
+void test_array_10(void);
+void test_array_11(void);
+void test_array_12(void);
+void test_array_13(void);
+void test_array_14(void);
+void test_array_15(void);
+void test_array_16(void);
+
+void test_all(void);
+
+void print_uvarchar()
+{
+ int i;
+
+ printf ("---->uvarchar variable,len=%d:\n",uvarchar_var.len);
+ for(i=0; i<VAR_SIZE; i++)
+ {
+ printf ("0x%04X ", uvarchar_var.arr[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_uvarchar_ind(int uvarchar_var_ind)
+{
+ printf("uvarchar_var_ind = %d\n",uvarchar_var_ind);
+}
+
+void print_local_uvarchar(utext *utext_var, int var_len)
+{
+ int i;
+
+ printf ("---->uvarchar variable,len=%d:\n",var_len);
+ for(i=0; i<20; i++)
+ {
+ printf ("0x%04X ", utext_var[i]);
+ if(i>6 && (i+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void print_array()
+{
+ int i,j;
+
+ for(i=0; i<ARRAY_SIZE; i++)
+ {
+ printf ("---->uvarchar array[%d]:(len=%d)\n", i,uvarchar_array[i].len);
+
+ for(j=0; j<VAR_SIZE; j++)
+ {
+ printf ("0x%04X ", uvarchar_array[i].arr[j]);
+ if(j>6 && (j+1)%8==0)
+ printf("\n");
+ }
+ printf("\n");
+ }
+
+ printf("\n");
+}
+
+int test_init()
+{
+ EXEC SQL CONNECT TO REGRESSDB1;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+ EXEC SQL WHENEVER SQLWARNING SQLPRINT;
+ EXEC SQL WHENEVER SQLERROR SQLPRINT;
+
+ EXEC SQL SET client_encoding='UTF8';
+
+ //initialization of test table
+ EXEC SQL CREATE TABLE tb1 (Item varchar, Count integer);
+
+ init_table_value();
+
+ return 0;
+}
+
+void test_finish()
+{
+ EXEC SQL DROP TABLE tb1;
+ EXEC SQL DISCONNECT ALL;
+}
+
+
+void init_table_value()
+{
+ EXEC SQL TRUNCATE tb1;
+
+ EXEC SQL INSERT INTO tb1 VALUES ('太𠮷𠜱平洋𠱓大西洋印度洋北冰洋', 1);
+ EXEC SQL INSERT INTO tb1 VALUES ('足球篮球羽毛球乒乓球橄榄球棒球冰球', 2);
+ EXEC SQL INSERT INTO tb1 VALUES ('世界杯每隔四年就会举行一次每次𠲖个球队', 3);
+ EXEC SQL INSERT INTO tb1 VALUES ('亚洲欧洲非洲大洋洲北美洲南美洲南极洲没有北极洲', 4);
+
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(8);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(9);
+ EXEC SQL INSERT INTO tb1 (Count) VALUES(10);
+}
+
+void init_var()
+{
+ int i;
+ memset((void*)&uvarchar_var,'a',sizeof(uvarchar_var));
+ uvarchar_var.len = 0;
+
+ memset((char*)uvarchar_array,'a',sizeof(uvarchar_array)*ARRAY_SIZE);
+
+ for(i=0;i<ARRAY_SIZE;i++)
+ {
+ uvarchar_array[i].len = 0;
+ }
+
+ uvarchar_var_ind = 0;
+}
+
+//simple select into uvarchar
+void test_var_1()
+{
+ test("test_var_1 : simple select into uvarchar var");
+ init_var();
+ //print_uvarchar();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=1;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=2;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=3;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=4;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+ init_var();
+}
+
+
+//simple select using uvarchar
+void test_var_2()
+{
+ test("test_var_2 : simple select using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=1;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:uvarchar_var;
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:uvarchar_var;
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using uvarchar
+void test_var_3()
+{
+ test("test_var_3 : simple update using uvarchar");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=1;
+ EXEC SQL UPDATE tb1 SET Item=:uvarchar_var WHERE Count=2;
+ EXEC SQL UPDATE tb1 SET Item=:uvarchar_var WHERE Count=3;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using uvarchar var
+void test_var_4()
+{
+ test("test_var_4 : simple delete using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=1;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:uvarchar_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:uvarchar_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using uvarchar var
+void test_var_5()
+{
+ test("test_var_5 : simple insert using uvarchar");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=1;
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_var, 11);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_var, 13);
+
+
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into uvarchar var
+void test_var_6()
+{
+ test("test_var_6 : prepared select into uvarchar var");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=?";
+
+ EXEC SQL EXECUTE stmt INTO :uvarchar_var USING 1;
+ print_uvarchar();
+
+ EXEC SQL EXECUTE stmt INTO :uvarchar_var USING 3;
+ print_uvarchar();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared select using uvarchar var
+void test_var_7()
+{
+ test("test_var_7 : prepared select using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :uvarchar_var;
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using uvarchar var
+void test_var_8()
+{
+ test("test_var_8 : prepared update using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :uvarchar_var, 1;
+ EXEC SQL EXECUTE stmt USING :uvarchar_var, 2;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using uvarchar var
+void test_var_9()
+{
+ test("test_var_9 : prepared delete using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :uvarchar_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using uvarchar var
+void test_var_10()
+{
+ test("test_var_10 : prepared insert using uvarchar var");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, 13)";
+ EXEC SQL EXECUTE stmt USING :uvarchar_var;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using uvarchar var
+void test_var_11()
+{
+ test("test_var_11 : Open cursor using uvarchar var");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :uvarchar_var FROM tb1 WHERE Count=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_var_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_var_11 USING :uvarchar_var;
+ EXEC SQL FETCH cursor_var_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_var_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into uvarchar var
+void test_var_12()
+{
+ test("test_var_12 : Fecth cursor into uvarchar var");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count=1";
+ EXEC SQL DECLARE cursor_var_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_var_12;
+ EXEC SQL FETCH cursor_var_12 INTO :uvarchar_var;
+ EXEC SQL CLOSE cursor_var_12;
+
+ print_uvarchar();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//simple insert using uvarchar with L string inited
+void test_var_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_var[VAR_SIZE];
+EXEC SQL END DECLARE SECTION;
+
+ test("test_var_13 : simple insert using uvarchar with L string inited");
+ init_var();
+
+ memset((char*)&uvarchar_local_var,0,sizeof(uvarchar_local_var));
+
+ memcpy((char*)uvarchar_local_var.arr, L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋", sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋"));
+ uvarchar_local_var.len = sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋")/4;
+
+ printf("uvarchar_local_var.len = %d\n",uvarchar_local_var.len);
+
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var, 16);
+
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL SELECT Item INTO :uvarchar_var:uvarchar_var_ind FROM tb1 WHERE Count=16;
+ print_uvarchar();
+ print_uvarchar_ind(uvarchar_var_ind);
+
+ init_table_value();
+}
+
+//Open cursor using utext var directly in WHERE Clause
+void test_var_14()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_var[20+1];
+EXEC SQL END DECLARE SECTION;
+ memset((char*)&uvarchar_local_var,0x00,sizeof(uvarchar_local_var));
+
+ test("test_var_14 : Open cursor using utext var directly in WHERE Clause");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :uvarchar_local_var FROM tb1 WHERE Count=3;
+ EXEC SQL DECLARE cursor_var_14 CURSOR FOR SELECT Count FROM tb1 WHERE Item=:uvarchar_local_var;
+ EXEC SQL OPEN cursor_var_14;
+ EXEC SQL FETCH cursor_var_14 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ EXEC SQL CLOSE cursor_var_14;
+
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Test uvarchar_var working with NULL without indicator
+void test_var_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_var[20+1];
+EXEC SQL END DECLARE SECTION;
+ test("test_var_15 : Test uvarchar_var working with NULL without indicator");
+
+ memset((char*)&uvarchar_local_var,'a',sizeof(uvarchar_local_var));
+ uvarchar_local_var.len=10;
+ EXEC SQL SELECT Item INTO :uvarchar_local_var FROM tb1 WHERE Count=8;
+ print_local_uvarchar(uvarchar_local_var.arr,20);
+
+
+ memset((char*)&uvarchar_local_var,0x00,sizeof(uvarchar_local_var));
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var, 18);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var, 19);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//uvarchar_var working with NULL with indicator
+void test_var_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_var[20+1];
+ int uvarchar_var_ind=-1;
+EXEC SQL END DECLARE SECTION;
+ test("test_var_16 : Test uvarchar_var working with NULL with indicator");
+
+ memset((char*)&uvarchar_local_var,'a',sizeof(uvarchar_local_var));
+ uvarchar_local_var.len=0;
+ EXEC SQL SELECT Item INTO :uvarchar_local_var:uvarchar_var_ind FROM tb1 WHERE Count=8;
+ print_local_uvarchar(uvarchar_local_var.arr,20);
+ print_uvarchar_ind(uvarchar_var_ind);
+
+ memset((char*)&uvarchar_local_var,0x00,sizeof(uvarchar_local_var));
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var:uvarchar_var_ind, 18);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var:uvarchar_var_ind, 19);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_var:uvarchar_var_ind, 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+
+//simple select into uvarchar array
+void test_array_1()
+{
+ test("test_array_1 : simple select into uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ print_array();
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array[0] FROM tb1 WHERE Count=1;
+ print_array();
+ init_var();
+}
+
+
+//simple select using uvarchar array
+void test_array_2()
+{
+ test("test_array_2 : simple select using uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:uvarchar_array[0];
+ printf ("count=%d for '太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT Count INTO :count FROM tb1 WHERE Item=:uvarchar_array[2];
+ printf ("count=%d for '世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+}
+
+//simple update using uvarchar array
+void test_array_3()
+{
+ test("test_array_3 : simple update using uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL UPDATE tb1 SET Item=:uvarchar_array[2] WHERE Count=1;
+
+ count = 0;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("find %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 0;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("find %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple delete using uvarchar array
+void test_array_4()
+{
+ test("test_array_4 : simple delete using uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ count = 100;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:uvarchar_array[0];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ count = 100;
+ EXEC SQL DELETE FROM tb1 WHERE Item=:uvarchar_array[2];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//simple insert using uvarchar array
+void test_array_5()
+{
+ test("test_array_5 : simple insert using uvarchar array");
+ init_var();
+
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array[0], 11);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array[2], 13);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+
+ init_table_value();
+}
+
+//prepared select into uvarchar array
+void test_array_6()
+{
+ test("test_array_6 : prepared select into uvarchar array");
+ init_var();
+
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count<=?";
+
+ EXEC SQL EXECUTE stmt INTO :uvarchar_array USING 3;
+
+ print_array();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared select using uvarchar array
+void test_array_7()
+{
+ test("test_array_7 : prepared select using uvarchar array");
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt INTO :count USING :uvarchar_array[0];
+
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+}
+
+//prepared update using uvarchar array
+void test_array_8()
+{
+ test("test_array_8 : prepared update using uvarchar array");
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "UPDATE tb1 SET Item=? WHERE Count=?";
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[0], 1;
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[0], 2;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared delete using uvarchar array
+void test_array_9()
+{
+ test("test_array_9 : prepared delete using uvarchar array");
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ count = 0;
+ EXEC SQL PREPARE stmt FROM "DELETE FROM tb1 WHERE Item=?";
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[0];
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋';
+
+ printf ("found %d rows for Item='太𠮷𠜱平洋𠱓大西洋印度洋北冰洋'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//prepared insert using uvarchar array
+void test_array_10()
+{
+ test("test_array_10 : prepared insert using uvarchar array");
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "INSERT INTO tb1 values (?, ?)";
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[2], 13;
+ EXEC SQL EXECUTE stmt USING :uvarchar_array[2], 15;
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item='世界杯每隔四年就会举行一次每次𠲖个球队';
+
+ printf ("found %d rows for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL DEALLOCATE PREPARE stmt;
+
+ init_table_value();
+}
+
+//Open cursor using uvarchar array
+void test_array_11()
+{
+ test("test_array_11 : Open cursor using uvarchar array");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=3;
+
+ EXEC SQL PREPARE stmt FROM "SELECT Count FROM tb1 WHERE Item=?";
+ EXEC SQL DECLARE cursor_array_11 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_array_11 USING :uvarchar_array[2];
+ EXEC SQL FETCH cursor_array_11 INTO :count;
+ printf ("count=%d for Item='世界杯每隔四年就会举行一次每次𠲖个球队'\n", count);
+ EXEC SQL CLOSE cursor_array_11;
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Fecth cursor into uvarchar array
+void test_array_12()
+{
+ test("test_array_12 : Fecth cursor into uvarchar array");
+ init_var();
+
+ EXEC SQL SET AUTOCOMMIT TO OFF;
+ EXEC SQL PREPARE stmt FROM "SELECT Item FROM tb1 WHERE Count<=3";
+ EXEC SQL DECLARE cursor_array_12 CURSOR FOR stmt;
+ EXEC SQL OPEN cursor_array_12;
+ EXEC SQL FETCH cursor_array_12 INTO :uvarchar_array[0];
+ EXEC SQL FETCH cursor_array_12 INTO :uvarchar_array[1];
+ EXEC SQL FETCH cursor_array_12 INTO :uvarchar_array[2];
+ EXEC SQL CLOSE cursor_array_12;
+
+ print_array();
+
+ EXEC SQL DEALLOCATE PREPARE stmt;
+ EXEC SQL SET AUTOCOMMIT TO ON;
+}
+
+//Insert array with L string inited
+void test_array_13()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_array13[3][VAR_SIZE];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_13 : Insert array with L string inited");
+
+ memset((char*)&uvarchar_array13,0,sizeof(uvarchar_array13));
+
+ memcpy((char*)uvarchar_array13[0].arr, L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋", sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋"));
+ uvarchar_array13[0].len = sizeof(L"太𠮷𠜱平洋𠱓大西洋印度洋北冰洋")/4;
+ memcpy((char*)uvarchar_array13[1].arr, L"足球篮球羽毛球乒乓球橄榄球棒球冰球", sizeof(L"足球篮球羽毛球乒乓球橄榄球棒球冰球"));
+ uvarchar_array13[1].len = sizeof(L"足球篮球羽毛球乒乓球橄榄球棒球冰球")/4;
+ memcpy((char*)uvarchar_array13[2].arr, L"世界杯每隔四年就会举行一次每次𠲖个球队", sizeof(L"世界杯每隔四年就会举行一次每次𠲖个球队"));
+ uvarchar_array13[2].len = sizeof(L"世界杯每隔四年就会举行一次每次𠲖个球队")/4;
+
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array13[0], 20);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array13[1], 21);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_array13[2], 22);
+
+ init_var();
+ EXEC SQL SELECT Item INTO :uvarchar_array FROM tb1 WHERE Count<=22 AND Count>=20;
+ print_array();
+ init_table_value();
+
+}
+
+//uvarchar array working with NULL without using indicator
+void test_array_15()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_array[3][20];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_15 : Test uvarchar array working with NULL without using indicator");
+ memset(uvarchar_local_array,'a',sizeof(uvarchar_local_array));
+
+ EXEC SQL SELECT Item INTO :uvarchar_local_array FROM tb1 WHERE Count>=8 and Count<=10;
+
+ print_local_uvarchar(uvarchar_local_array[0].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[1].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[2].arr, 20);
+
+
+ memset(uvarchar_local_array,0x00,sizeof(uvarchar_local_array));
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[1], 19);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[2], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+//uvarchar array working with NULL using indicator
+void test_array_16()
+{
+EXEC SQL BEGIN DECLARE SECTION;
+ uvarchar uvarchar_local_array[3][20];
+ int uvarchar_array_ind[3];
+EXEC SQL END DECLARE SECTION;
+ test("test_array_16 : Test uvarchar array working with NULL using indicator");
+ memset(uvarchar_local_array,'a',sizeof(uvarchar_local_array));
+
+ EXEC SQL SELECT Item INTO :uvarchar_local_array:uvarchar_array_ind FROM tb1 WHERE Count>=8 and Count<=10;
+
+
+ print_local_uvarchar(uvarchar_local_array[0].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[1].arr, 20);
+ print_local_uvarchar(uvarchar_local_array[2].arr, 20);
+
+ print_uvarchar_ind(uvarchar_array_ind[0]);
+ print_uvarchar_ind(uvarchar_array_ind[1]);
+ print_uvarchar_ind(uvarchar_array_ind[2]);
+
+ memset(uvarchar_local_array,0x00,sizeof(uvarchar_local_array));
+ uvarchar_array_ind[0]=-1;
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[0]:uvarchar_array_ind[0], 18);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[1]:uvarchar_array_ind[0], 19);
+ EXEC SQL INSERT INTO tb1 values (:uvarchar_local_array[2]:uvarchar_array_ind[0], 20);
+ EXEC SQL SELECT count(*) INTO :count FROM tb1 WHERE Item is NULL;
+ printf ("found %d rows for Item being NULL\n", count);
+
+ init_table_value();
+}
+
+void test_all()
+{
+ test_var_1();
+ test_var_2();
+ test_var_3();
+ test_var_4();
+ test_var_5();
+ test_var_6();
+ test_var_7();
+ test_var_8();
+ test_var_9();
+ test_var_10();
+ test_var_11();
+ test_var_12();
+ test_var_13();
+ test_var_14();
+ test_var_15();
+ test_var_16();
+ test_array_1();
+ test_array_2();
+ test_array_3();
+ test_array_4();
+ test_array_5();
+ test_array_6();
+ test_array_7();
+ test_array_8();
+ test_array_9();
+ test_array_10();
+ test_array_11();
+ test_array_12();
+ test_array_13();
+ test_array_15();
+ test_array_16();
+}
+
+int main()
+{
+// ECPGdebug(1, stderr);
+ if(test_init() !=0)
+ return -1;
+
+ test_all();
+ test_finish();
+
+ return 0;
+}
diff --git a/src/interfaces/ecpg/test/unicode/uvarchar.sh b/src/interfaces/ecpg/test/unicode/uvarchar.sh
new file mode 100755
index 0000000..4d05ed0
--- /dev/null
+++ b/src/interfaces/ecpg/test/unicode/uvarchar.sh
@@ -0,0 +1,11 @@
+echo "../../preproc/ecpg --regression --enable-uvarchar -I./../../include -I. -o uvarchar.c uvarchar.pgc"
+
+../../preproc/ecpg --regression -r no_indicator --enable-utext -I./../../include -I. -o uvarchar.c uvarchar.pgc
+
+gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -O0 -pthread -D_REENTRANT -D_THREAD_SAFE -D_POSIX_PTHREAD_SEMANTICS -I../../include -I../../../../../src/interfaces/ecpg/include -I../../../../../src/interfaces/libpq -I../../../../../src/include -DLINUX_OOM_SCORE_ADJ=0 -DLINUX_OOM_ADJ=0 -D_GNU_SOURCE -I/db/pgmaster/run/include -g -c -o uvarchar.o uvarchar.c
+
+
+gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -O0 -pthread -D_REENTRANT -D_THREAD_SAFE -D_POSIX_PTHREAD_SEMANTICS uvarchar.o -L../../ecpglib -L../../pgtypeslib -L../../../../../src/interfaces/libpq -L../../../../../src/port -L../../../../../src/common -L/db/pgmaster/run/lib -Wl,--as-needed -Wl,-rpath,--enable-new-dtags -lecpg -lpgtypes -lpq -lpgcommon -lpgport -lselinux -lxslt -lxml2 -lpam -lssl -lcrypto -lgssapi_krb5 -lz -lrt -lcrypt -ldl -lm -g -o uvarchar
+
+
+./uvarchar >results/uvarchar.ret 2>&1
diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt
index d6a38d0..8be711d 100644
--- a/src/interfaces/libpq/exports.txt
+++ b/src/interfaces/libpq/exports.txt
@@ -172,3 +172,8 @@ PQsslAttribute 169
PQsetErrorContextVisibility 170
PQresultVerboseErrorMessage 171
PQencryptPasswordConn 172
+PQGetEncodingFromPGres 173
+PQGetEncodingFromPGconn 174
+PQGetEncodingName 175
+PQGenEncodingMaxLen 176
+
diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c
index c24bce6..3a1ca04 100644
--- a/src/interfaces/libpq/fe-exec.c
+++ b/src/interfaces/libpq/fe-exec.c
@@ -3813,3 +3813,54 @@ PQunescapeBytea(const unsigned char *strtext, size_t *retbuflen)
*retbuflen = buflen;
return tmpbuf;
}
+
+/*
+ * New added interface for ECOBPG NCHAR features
+ * to get client_encoding in src/interface/ecpg/ecpglib/data.c
+ * src/interface/ecpg/ecpglib/execute.c.
+ */
+
+int
+PQGetEncodingFromPGconn(const PGconn *conn)
+{
+ return conn->client_encoding;
+}
+
+int
+PQGetEncodingFromPGres(const PGresult *results)
+{
+ return results->client_encoding;
+}
+
+int
+PQGetencUTF8(void)
+{
+ return PG_UTF8;
+}
+
+int
+PQGetencEUCJP(void)
+{
+ return PG_EUC_JP;
+}
+
+char *
+PQGetEncodingName(int client_encoding)
+{
+ int i;
+
+ for (i = 0; pg_enc2gettext_tbl[i].name != NULL; i++)
+ {
+ if (pg_enc2gettext_tbl[i].encoding == client_encoding)
+ return pg_enc2gettext_tbl[i].name;
+ }
+
+ /* Should not go to here actually */
+ return NULL;
+}
+
+int
+PQGenEncodingMaxLen(int client_encoding)
+{
+ return pg_encmaxlen_tbl[client_encoding].len;
+}
diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h
index 1d915e7..f5fcb6e 100644
--- a/src/interfaces/libpq/libpq-fe.h
+++ b/src/interfaces/libpq/libpq-fe.h
@@ -535,7 +535,23 @@ extern size_t PQescapeString(char *to, const char *from, size_t length);
extern unsigned char *PQescapeBytea(const unsigned char *from, size_t from_length,
size_t *to_length);
+/*
+ * New added interface for ECOBPG NCHAR features
+ * to get client_encoding in src/interface/ecpg/ecpglib/data.c
+ * src/interface/ecpg/ecpglib/execute.c.
+ */
+
+extern int PQGetEncodingFromPGconn(const PGconn *conn);
+
+extern int PQGetEncodingFromPGres(const PGresult *results);
+
+extern char *PQGetEncodingName(int client_encoding);
+
+extern int PQGenEncodingMaxLen(int client_encoding);
+
+extern int PQGetencUTF8(void);
+extern int PQGetencEUCJP(void);
/* === in fe-print.c === */
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v5 1/2] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 359 insertions(+), 553 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 39bea204d16..c99a0de4ddb 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1887,12 +1868,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1901,7 +1882,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1943,9 +1924,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2132,7 +2113,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2597,7 +2578,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2693,7 +2674,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2709,7 +2690,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2884,67 +2865,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2952,7 +2872,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2961,13 +2881,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2986,11 +2906,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3029,12 +2948,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3050,15 +2969,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3072,42 +2990,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3164,8 +3081,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3177,10 +3093,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3227,14 +3142,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index debf12e1b0b..6d7f862fb5c 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenFileSet(&fileset->fs, filename, O_RDONLY, false);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenFileSet() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -731,26 +607,145 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
}
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenFileSet(<s->fileset->fs, filename, O_RDONLY, false);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +767,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +810,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +852,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +896,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +928,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +987,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1012,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1039,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1054,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1068,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1106,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1139,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1168,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,13 +1191,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
-
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index b17347b2141..d5930f258d9 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -888,7 +889,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2221,7 +2222,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2254,8 +2255,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2269,20 +2269,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2299,15 +2297,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2365,11 +2362,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2667,9 +2663,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2919,7 +2918,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2981,11 +2980,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -3013,18 +3015,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3037,7 +3042,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3080,7 +3086,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3146,17 +3152,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3173,6 +3180,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3239,10 +3247,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3255,7 +3263,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3289,9 +3297,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3322,8 +3328,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3354,8 +3359,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3697,11 +3701,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3710,11 +3714,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3892,7 +3896,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3903,13 +3907,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3920,7 +3921,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3930,11 +3931,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4135,21 +4134,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4160,7 +4155,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4169,16 +4164,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4392,19 +4384,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4415,16 +4404,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4466,7 +4453,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4491,13 +4478,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4508,7 +4492,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4522,8 +4506,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4531,16 +4514,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4652,7 +4633,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4668,7 +4649,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4687,9 +4668,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4733,9 +4714,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 37cb4f3d59a..2e8cbee69ff 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -41,6 +41,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2316,7 +2317,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.30.2
--------------5B6EE213D46233186D258C9F
Content-Type: text/x-patch; charset=UTF-8;
name="v5-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v5-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH 1/2] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 457 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 553 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 914b02ceee4..80025e5f1f3 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1887,12 +1868,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1901,7 +1882,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1943,9 +1924,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2128,7 +2109,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2593,7 +2574,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2689,7 +2670,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2705,7 +2686,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2880,67 +2861,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2948,7 +2868,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2957,13 +2877,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2982,11 +2902,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3025,12 +2944,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3046,15 +2965,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3068,42 +2986,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3160,8 +3077,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3173,10 +3089,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3223,14 +3138,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index cafc0872549..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,13 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
-
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index b17347b2141..d5930f258d9 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -888,7 +889,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2221,7 +2222,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2254,8 +2255,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2269,20 +2269,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2299,15 +2297,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2365,11 +2362,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2667,9 +2663,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2919,7 +2918,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2981,11 +2980,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -3013,18 +3015,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3037,7 +3042,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3080,7 +3086,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3146,17 +3152,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3173,6 +3180,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3239,10 +3247,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3255,7 +3263,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3289,9 +3297,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3322,8 +3328,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3354,8 +3359,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3697,11 +3701,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3710,11 +3714,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3892,7 +3896,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3903,13 +3907,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3920,7 +3921,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3930,11 +3931,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4135,21 +4134,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4160,7 +4155,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4169,16 +4164,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4392,19 +4384,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4415,16 +4404,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4466,7 +4453,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4491,13 +4478,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4508,7 +4492,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4522,8 +4506,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4531,16 +4514,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4652,7 +4633,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4668,7 +4649,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4687,9 +4668,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4733,9 +4714,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 105180764e1..cd1452ec9f4 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -41,6 +41,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2315,7 +2316,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.30.2
--------------EDB671A517E7E589D72F13BA
Content-Type: text/x-patch; charset=UTF-8;
name="0002-Replace-polyphase-merge-algorithm-with-a-simple-bala.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="0002-Replace-polyphase-merge-algorithm-with-a-simple-bala.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v2 1/2] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 75e5bbf209d..f680f74ed5a 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1881,12 +1862,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1895,7 +1876,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1937,9 +1918,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2123,7 +2104,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2588,7 +2569,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2680,7 +2661,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2696,7 +2677,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2871,67 +2852,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2939,7 +2859,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2948,13 +2868,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2973,11 +2893,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3016,12 +2935,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3037,15 +2956,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3059,42 +2977,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3151,8 +3068,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3164,10 +3080,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3214,14 +3129,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 28905124f96..ecefa9bac96 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index d0cc04a878a..07b6c92f54d 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 6c0a7d68d61..16378209f08 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2183,7 +2184,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index da5159e4c6c..97bc967902d 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.20.1
--------------A0613037A47B3A76097E426B
Content-Type: text/x-patch; charset=UTF-8;
name="v2-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v2-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface.
@ 2020-10-21 18:57 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-10-21 18:57 UTC (permalink / raw)
All the tape functions, like LogicalTapeRead and LogicalTapeWrite, now
take a LogicalTape as argument, instead of LogicalTapeSet+tape number.
You can create any number of LogicalTapes in a single LogicalTapeSet, and
you don't need to decide the number upfront, when you create the tape set.
This makes the tape management in hash agg spilling in nodeAgg.c simpler.
---
src/backend/executor/nodeAgg.c | 187 ++++--------
src/backend/utils/sort/logtape.c | 456 ++++++++++++-----------------
src/backend/utils/sort/tuplesort.c | 229 +++++++--------
src/include/nodes/execnodes.h | 3 +-
src/include/utils/logtape.h | 37 ++-
5 files changed, 360 insertions(+), 552 deletions(-)
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03f..13f6f188668 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -208,7 +208,16 @@
*
* Spilled data is written to logical tapes. These provide better control
* over memory usage, disk space, and the number of files than if we were
- * to use a BufFile for each spill.
+ * to use a BufFile for each spill. We don't know the number of tapes needed
+ * at the start of the algorithm (because it can recurse), so a tape set is
+ * allocated at the beginning, and individual tapes are created as needed.
+ * As a particular tape is read, logtape.c recycles its disk space. When a
+ * tape is read to completion, it is destroyed entirely.
+ *
+ * Tapes' buffers can take up substantial memory when many tapes are open at
+ * once. We only need one tape open at a time in read mode (using a buffer
+ * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
+ * requiring a buffer of size BLCKSZ) for each partition.
*
* Note that it's possible for transition states to start small but then
* grow very large; for instance in the case of ARRAY_AGG. In such cases,
@@ -311,27 +320,6 @@
*/
#define CHUNKHDRSZ 16
-/*
- * Track all tapes needed for a HashAgg that spills. We don't know the maximum
- * number of tapes needed at the start of the algorithm (because it can
- * recurse), so one tape set is allocated and extended as needed for new
- * tapes. When a particular tape is already read, rewind it for write mode and
- * put it in the free list.
- *
- * Tapes' buffers can take up substantial memory when many tapes are open at
- * once. We only need one tape open at a time in read mode (using a buffer
- * that's a multiple of BLCKSZ); but we need one tape open in write mode (each
- * requiring a buffer of size BLCKSZ) for each partition.
- */
-typedef struct HashTapeInfo
-{
- LogicalTapeSet *tapeset;
- int ntapes;
- int *freetapes;
- int nfreetapes;
- int freetapes_alloc;
-} HashTapeInfo;
-
/*
* Represents partitioned spill data for a single hashtable. Contains the
* necessary information to route tuples to the correct partition, and to
@@ -343,9 +331,8 @@ typedef struct HashTapeInfo
*/
typedef struct HashAggSpill
{
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
int npartitions; /* number of partitions */
- int *partitions; /* spill partition tape numbers */
+ LogicalTape **partitions; /* spill partition tapes */
int64 *ntuples; /* number of tuples in each partition */
uint32 mask; /* mask to find partition from hash value */
int shift; /* after masking, shift by this amount */
@@ -365,8 +352,7 @@ typedef struct HashAggBatch
{
int setno; /* grouping set */
int used_bits; /* number of bits of hash already used */
- LogicalTapeSet *tapeset; /* borrowed reference to tape set */
- int input_tapenum; /* input partition tape */
+ LogicalTape *input_tape; /* input partition tape */
int64 input_tuples; /* number of tuples in this batch */
double input_card; /* estimated group cardinality */
} HashAggBatch;
@@ -442,22 +428,17 @@ static void hash_agg_update_metrics(AggState *aggstate, bool from_tape,
int npartitions);
static void hashagg_finish_initial_spills(AggState *aggstate);
static void hashagg_reset_spill_state(AggState *aggstate);
-static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
- int input_tapenum, int setno,
+static HashAggBatch *hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card,
int used_bits);
static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
-static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
+static void hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *lts,
int used_bits, double input_groups,
double hashentrysize);
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *slot, uint32 hash);
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
int setno);
-static void hashagg_tapeinfo_init(AggState *aggstate);
-static void hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *dest,
- int ndest);
-static void hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum);
static Datum GetAggInitVal(Datum textInitVal, Oid transtype);
static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
AggState *aggstate, EState *estate,
@@ -1879,12 +1860,12 @@ hash_agg_enter_spill_mode(AggState *aggstate)
if (!aggstate->hash_ever_spilled)
{
- Assert(aggstate->hash_tapeinfo == NULL);
+ Assert(aggstate->hash_tapeset == NULL);
Assert(aggstate->hash_spills == NULL);
aggstate->hash_ever_spilled = true;
- hashagg_tapeinfo_init(aggstate);
+ aggstate->hash_tapeset = LogicalTapeSetCreate(true, NULL, -1);
aggstate->hash_spills = palloc(sizeof(HashAggSpill) * aggstate->num_hashes);
@@ -1893,7 +1874,7 @@ hash_agg_enter_spill_mode(AggState *aggstate)
AggStatePerHash perhash = &aggstate->perhash[setno];
HashAggSpill *spill = &aggstate->hash_spills[setno];
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
}
@@ -1935,9 +1916,9 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions)
aggstate->hash_mem_peak = total_mem;
/* update disk usage */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeinfo->tapeset) * (BLCKSZ / 1024);
+ uint64 disk_used = LogicalTapeSetBlocks(aggstate->hash_tapeset) * (BLCKSZ / 1024);
if (aggstate->hash_disk_used < disk_used)
aggstate->hash_disk_used = disk_used;
@@ -2121,7 +2102,7 @@ lookup_hash_entries(AggState *aggstate)
TupleTableSlot *slot = aggstate->tmpcontext->ecxt_outertuple;
if (spill->partitions == NULL)
- hashagg_spill_init(spill, aggstate->hash_tapeinfo, 0,
+ hashagg_spill_init(spill, aggstate->hash_tapeset, 0,
perhash->aggnode->numGroups,
aggstate->hashentrysize);
@@ -2586,7 +2567,7 @@ agg_refill_hash_table(AggState *aggstate)
HashAggBatch *batch;
AggStatePerHash perhash;
HashAggSpill spill;
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
+ LogicalTapeSet *tapeset = aggstate->hash_tapeset;
bool spill_initialized = false;
if (aggstate->hash_batches == NIL)
@@ -2682,7 +2663,7 @@ agg_refill_hash_table(AggState *aggstate)
* that we don't assign tapes that will never be used.
*/
spill_initialized = true;
- hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
+ hashagg_spill_init(&spill, tapeset, batch->used_bits,
batch->input_card, aggstate->hashentrysize);
}
/* no memory for a new group, spill */
@@ -2698,7 +2679,7 @@ agg_refill_hash_table(AggState *aggstate)
ResetExprContext(aggstate->tmpcontext);
}
- hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
+ LogicalTapeClose(batch->input_tape);
/* change back to phase 0 */
aggstate->current_phase = 0;
@@ -2873,67 +2854,6 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate)
return NULL;
}
-/*
- * Initialize HashTapeInfo
- */
-static void
-hashagg_tapeinfo_init(AggState *aggstate)
-{
- HashTapeInfo *tapeinfo = palloc(sizeof(HashTapeInfo));
- int init_tapes = 16; /* expanded dynamically */
-
- tapeinfo->tapeset = LogicalTapeSetCreate(init_tapes, true, NULL, NULL, -1);
- tapeinfo->ntapes = init_tapes;
- tapeinfo->nfreetapes = init_tapes;
- tapeinfo->freetapes_alloc = init_tapes;
- tapeinfo->freetapes = palloc(init_tapes * sizeof(int));
- for (int i = 0; i < init_tapes; i++)
- tapeinfo->freetapes[i] = i;
-
- aggstate->hash_tapeinfo = tapeinfo;
-}
-
-/*
- * Assign unused tapes to spill partitions, extending the tape set if
- * necessary.
- */
-static void
-hashagg_tapeinfo_assign(HashTapeInfo *tapeinfo, int *partitions,
- int npartitions)
-{
- int partidx = 0;
-
- /* use free tapes if available */
- while (partidx < npartitions && tapeinfo->nfreetapes > 0)
- partitions[partidx++] = tapeinfo->freetapes[--tapeinfo->nfreetapes];
-
- if (partidx < npartitions)
- {
- LogicalTapeSetExtend(tapeinfo->tapeset, npartitions - partidx);
-
- while (partidx < npartitions)
- partitions[partidx++] = tapeinfo->ntapes++;
- }
-}
-
-/*
- * After a tape has already been written to and then read, this function
- * rewinds it for writing and adds it to the free list.
- */
-static void
-hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
-{
- /* rewinding frees the buffer while not in use */
- LogicalTapeRewindForWrite(tapeinfo->tapeset, tapenum);
- if (tapeinfo->freetapes_alloc == tapeinfo->nfreetapes)
- {
- tapeinfo->freetapes_alloc <<= 1;
- tapeinfo->freetapes = repalloc(tapeinfo->freetapes,
- tapeinfo->freetapes_alloc * sizeof(int));
- }
- tapeinfo->freetapes[tapeinfo->nfreetapes++] = tapenum;
-}
-
/*
* hashagg_spill_init
*
@@ -2941,7 +2861,7 @@ hashagg_tapeinfo_release(HashTapeInfo *tapeinfo, int tapenum)
* of partitions to create, and initializes them.
*/
static void
-hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
+hashagg_spill_init(HashAggSpill *spill, LogicalTapeSet *tapeset, int used_bits,
double input_groups, double hashentrysize)
{
int npartitions;
@@ -2950,13 +2870,13 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
npartitions = hash_choose_num_partitions(input_groups, hashentrysize,
used_bits, &partition_bits);
- spill->partitions = palloc0(sizeof(int) * npartitions);
+ spill->partitions = palloc0(sizeof(LogicalTape *) * npartitions);
spill->ntuples = palloc0(sizeof(int64) * npartitions);
spill->hll_card = palloc0(sizeof(hyperLogLogState) * npartitions);
- hashagg_tapeinfo_assign(tapeinfo, spill->partitions, npartitions);
+ for (int i = 0; i < npartitions; i++)
+ spill->partitions[i] = LogicalTapeCreate(tapeset);
- spill->tapeset = tapeinfo->tapeset;
spill->shift = 32 - used_bits - partition_bits;
spill->mask = (npartitions - 1) << spill->shift;
spill->npartitions = npartitions;
@@ -2975,11 +2895,10 @@ static Size
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
TupleTableSlot *inputslot, uint32 hash)
{
- LogicalTapeSet *tapeset = spill->tapeset;
TupleTableSlot *spillslot;
int partition;
MinimalTuple tuple;
- int tapenum;
+ LogicalTape *tape;
int total_written = 0;
bool shouldFree;
@@ -3018,12 +2937,12 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
*/
addHyperLogLog(&spill->hll_card[partition], hash_bytes_uint32(hash));
- tapenum = spill->partitions[partition];
+ tape = spill->partitions[partition];
- LogicalTapeWrite(tapeset, tapenum, (void *) &hash, sizeof(uint32));
+ LogicalTapeWrite(tape, (void *) &hash, sizeof(uint32));
total_written += sizeof(uint32);
- LogicalTapeWrite(tapeset, tapenum, (void *) tuple, tuple->t_len);
+ LogicalTapeWrite(tape, (void *) tuple, tuple->t_len);
total_written += tuple->t_len;
if (shouldFree)
@@ -3039,15 +2958,14 @@ hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
* be done.
*/
static HashAggBatch *
-hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
+hashagg_batch_new(LogicalTape *input_tape, int setno,
int64 input_tuples, double input_card, int used_bits)
{
HashAggBatch *batch = palloc0(sizeof(HashAggBatch));
batch->setno = setno;
batch->used_bits = used_bits;
- batch->tapeset = tapeset;
- batch->input_tapenum = tapenum;
+ batch->input_tape = input_tape;
batch->input_tuples = input_tuples;
batch->input_card = input_card;
@@ -3061,42 +2979,41 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
static MinimalTuple
hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
{
- LogicalTapeSet *tapeset = batch->tapeset;
- int tapenum = batch->input_tapenum;
+ LogicalTape *tape = batch->input_tape;
MinimalTuple tuple;
uint32 t_len;
size_t nread;
uint32 hash;
- nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
+ nread = LogicalTapeRead(tape, &hash, sizeof(uint32));
if (nread == 0)
return NULL;
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
if (hashp != NULL)
*hashp = hash;
- nread = LogicalTapeRead(tapeset, tapenum, &t_len, sizeof(t_len));
+ nread = LogicalTapeRead(tape, &t_len, sizeof(t_len));
if (nread != sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, sizeof(uint32), nread)));
tuple = (MinimalTuple) palloc(t_len);
tuple->t_len = t_len;
- nread = LogicalTapeRead(tapeset, tapenum,
+ nread = LogicalTapeRead(tape,
(void *) ((char *) tuple + sizeof(uint32)),
t_len - sizeof(uint32));
if (nread != t_len - sizeof(uint32))
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
- tapenum, t_len - sizeof(uint32), nread)));
+ errmsg("unexpected EOF for tape %p: requested %zu bytes, read %zu bytes",
+ tape, t_len - sizeof(uint32), nread)));
return tuple;
}
@@ -3153,8 +3070,7 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
for (i = 0; i < spill->npartitions; i++)
{
- LogicalTapeSet *tapeset = aggstate->hash_tapeinfo->tapeset;
- int tapenum = spill->partitions[i];
+ LogicalTape *tape = spill->partitions[i];
HashAggBatch *new_batch;
double cardinality;
@@ -3166,10 +3082,9 @@ hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill, int setno)
freeHyperLogLog(&spill->hll_card[i]);
/* rewinding frees the buffer while not in use */
- LogicalTapeRewindForRead(tapeset, tapenum,
- HASHAGG_READ_BUFFER_SIZE);
+ LogicalTapeRewindForRead(tape, HASHAGG_READ_BUFFER_SIZE);
- new_batch = hashagg_batch_new(tapeset, tapenum, setno,
+ new_batch = hashagg_batch_new(tape, setno,
spill->ntuples[i], cardinality,
used_bits);
aggstate->hash_batches = lcons(new_batch, aggstate->hash_batches);
@@ -3216,14 +3131,10 @@ hashagg_reset_spill_state(AggState *aggstate)
aggstate->hash_batches = NIL;
/* close tape set */
- if (aggstate->hash_tapeinfo != NULL)
+ if (aggstate->hash_tapeset != NULL)
{
- HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
-
- LogicalTapeSetClose(tapeinfo->tapeset);
- pfree(tapeinfo->freetapes);
- pfree(tapeinfo);
- aggstate->hash_tapeinfo = NULL;
+ LogicalTapeSetClose(aggstate->hash_tapeset);
+ aggstate->hash_tapeset = NULL;
}
}
diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c
index 089ba2e1068..82812716c2d 100644
--- a/src/backend/utils/sort/logtape.c
+++ b/src/backend/utils/sort/logtape.c
@@ -9,8 +9,7 @@
* there is an annoying problem: the peak space usage is at least twice
* the volume of actual data to be sorted. (This must be so because each
* datum will appear in both the input and output tapes of the final
- * merge pass. For seven-tape polyphase merge, which is otherwise a
- * pretty good algorithm, peak usage is more like 4x actual data volume.)
+ * merge pass.)
*
* We can work around this problem by recognizing that any one tape
* dataset (with the possible exception of the final output) is written
@@ -137,6 +136,8 @@ typedef struct TapeBlockTrailer
*/
typedef struct LogicalTape
{
+ LogicalTapeSet *tapeSet; /* tape set this tape is part of */
+
bool writing; /* T while in write phase */
bool frozen; /* T if blocks should not be freed when read */
bool dirty; /* does buffer need to be written? */
@@ -180,11 +181,14 @@ typedef struct LogicalTape
* This data structure represents a set of related "logical tapes" sharing
* space in a single underlying file. (But that "file" may be multiple files
* if needed to escape OS limits on file size; buffile.c handles that for us.)
- * The number of tapes is fixed at creation.
+ * Tapes belonging to a tape set can be created and destroyed on-the-fly, on
+ * demand.
*/
struct LogicalTapeSet
{
BufFile *pfile; /* underlying file for whole tape set */
+ SharedFileSet *fileset;
+ int worker; /* worker # if shared, -1 for leader/serial */
/*
* File size tracking. nBlocksWritten is the size of the underlying file,
@@ -213,22 +217,16 @@ struct LogicalTapeSet
long nFreeBlocks; /* # of currently free blocks */
Size freeBlocksLen; /* current allocated length of freeBlocks[] */
bool enable_prealloc; /* preallocate write blocks? */
-
- /* The array of logical tapes. */
- int nTapes; /* # of logical tapes in set */
- LogicalTape *tapes; /* has nTapes nentries */
};
+static LogicalTape *ltsCreateTape(LogicalTapeSet *lts);
static void ltsWriteBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static void ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer);
static long ltsGetBlock(LogicalTapeSet *lts, LogicalTape *lt);
static long ltsGetFreeBlock(LogicalTapeSet *lts);
static long ltsGetPreallocBlock(LogicalTapeSet *lts, LogicalTape *lt);
static void ltsReleaseBlock(LogicalTapeSet *lts, long blocknum);
-static void ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset);
-static void ltsInitTape(LogicalTape *lt);
-static void ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt);
+static void ltsInitReadBuffer(LogicalTape *lt);
/*
@@ -304,7 +302,7 @@ ltsReadBlock(LogicalTapeSet *lts, long blocknum, void *buffer)
* Returns true if anything was read, 'false' on EOF.
*/
static bool
-ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsReadFillBuffer(LogicalTape *lt)
{
lt->pos = 0;
lt->nbytes = 0;
@@ -321,9 +319,9 @@ ltsReadFillBuffer(LogicalTapeSet *lts, LogicalTape *lt)
datablocknum += lt->offsetBlockNumber;
/* Read the block */
- ltsReadBlock(lts, datablocknum, (void *) thisbuf);
+ ltsReadBlock(lt->tapeSet, datablocknum, (void *) thisbuf);
if (!lt->frozen)
- ltsReleaseBlock(lts, datablocknum);
+ ltsReleaseBlock(lt->tapeSet, datablocknum);
lt->curBlockNumber = lt->nextBlockNumber;
lt->nbytes += TapeBlockGetNBytes(thisbuf);
@@ -530,126 +528,13 @@ ltsReleaseBlock(LogicalTapeSet *lts, long blocknum)
}
}
-/*
- * Claim ownership of a set of logical tapes from existing shared BufFiles.
- *
- * Caller should be leader process. Though tapes are marked as frozen in
- * workers, they are not frozen when opened within leader, since unfrozen tapes
- * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
- * for random access.)
- */
-static void
-ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
- SharedFileSet *fileset)
-{
- LogicalTape *lt = NULL;
- long tapeblocks = 0L;
- long nphysicalblocks = 0L;
- int i;
-
- /* Should have at least one worker tape, plus leader's tape */
- Assert(lts->nTapes >= 2);
-
- /*
- * Build concatenated view of all BufFiles, remembering the block number
- * where each source file begins. No changes are needed for leader/last
- * tape.
- */
- for (i = 0; i < lts->nTapes - 1; i++)
- {
- char filename[MAXPGPATH];
- BufFile *file;
- int64 filesize;
-
- lt = <s->tapes[i];
-
- pg_itoa(i, filename);
- file = BufFileOpenShared(fileset, filename, O_RDONLY);
- filesize = BufFileSize(file);
-
- /*
- * Stash first BufFile, and concatenate subsequent BufFiles to that.
- * Store block offset into each tape as we go.
- */
- lt->firstBlockNumber = shared[i].firstblocknumber;
- if (i == 0)
- {
- lts->pfile = file;
- lt->offsetBlockNumber = 0L;
- }
- else
- {
- lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
- }
- /* Don't allocate more for read buffer than could possibly help */
- lt->max_size = Min(MaxAllocSize, filesize);
- tapeblocks = filesize / BLCKSZ;
- nphysicalblocks += tapeblocks;
- }
-
- /*
- * Set # of allocated blocks, as well as # blocks written. Use extent of
- * new BufFile space (from 0 to end of last worker's tape space) for this.
- * Allocated/written blocks should include space used by holes left
- * between concatenated BufFiles.
- */
- lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
- lts->nBlocksWritten = lts->nBlocksAllocated;
-
- /*
- * Compute number of hole blocks so that we can later work backwards, and
- * instrument number of physical blocks. We don't simply use physical
- * blocks directly for instrumentation because this would break if we ever
- * subsequently wrote to the leader tape.
- *
- * Working backwards like this keeps our options open. If shared BufFiles
- * ever support being written to post-export, logtape.c can automatically
- * take advantage of that. We'd then support writing to the leader tape
- * while recycling space from worker tapes, because the leader tape has a
- * zero offset (write routines won't need to have extra logic to apply an
- * offset).
- *
- * The only thing that currently prevents writing to the leader tape from
- * working is the fact that BufFiles opened using BufFileOpenShared() are
- * read-only by definition, but that could be changed if it seemed
- * worthwhile. For now, writing to the leader tape will raise a "Bad file
- * descriptor" error, so tuplesort must avoid writing to the leader tape
- * altogether.
- */
- lts->nHoleBlocks = lts->nBlocksAllocated - nphysicalblocks;
-}
-
-/*
- * Initialize per-tape struct. Note we allocate the I/O buffer lazily.
- */
-static void
-ltsInitTape(LogicalTape *lt)
-{
- lt->writing = true;
- lt->frozen = false;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->nextBlockNumber = -1L;
- lt->offsetBlockNumber = 0L;
- lt->buffer = NULL;
- lt->buffer_size = 0;
- /* palloc() larger than MaxAllocSize would fail */
- lt->max_size = MaxAllocSize;
- lt->pos = 0;
- lt->nbytes = 0;
- lt->prealloc = NULL;
- lt->nprealloc = 0;
- lt->prealloc_size = 0;
-}
-
/*
* Lazily allocate and initialize the read buffer. This avoids waste when many
* tapes are open at once, but not all are active between rewinding and
* reading.
*/
static void
-ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
+ltsInitReadBuffer(LogicalTape *lt)
{
Assert(lt->buffer_size > 0);
lt->buffer = palloc(lt->buffer_size);
@@ -658,40 +543,32 @@ ltsInitReadBuffer(LogicalTapeSet *lts, LogicalTape *lt)
lt->nextBlockNumber = lt->firstBlockNumber;
lt->pos = 0;
lt->nbytes = 0;
- ltsReadFillBuffer(lts, lt);
+ ltsReadFillBuffer(lt);
}
/*
- * Create a set of logical tapes in a temporary underlying file.
+ * Create a tape set, backed by a temporary underlying file.
*
- * Each tape is initialized in write state. Serial callers pass ntapes,
- * NULL argument for shared, and -1 for worker. Parallel worker callers
- * pass ntapes, a shared file handle, NULL shared argument, and their own
- * worker number. Leader callers, which claim shared worker tapes here,
- * must supply non-sentinel values for all arguments except worker number,
- * which should be -1.
+ * The tape set is initially empty. Use LogicalTapeCreate() to create
+ * tapes in it.
*
- * Leader caller is passing back an array of metadata each worker captured
- * when LogicalTapeFreeze() was called for their final result tapes. Passed
- * tapes array is actually sized ntapes - 1, because it includes only
- * worker tapes, whereas leader requires its own leader tape. Note that we
- * rely on the assumption that reclaimed worker tapes will only be read
- * from once by leader, and never written to again (tapes are initialized
- * for writing, but that's only to be consistent). Leader may not write to
- * its own tape purely due to a restriction in the shared buffile
- * infrastructure that may be lifted in the future.
+ * Serial callers pass NULL argument for shared, and -1 for worker. Parallel
+ * worker callers pass a shared file handle and their own worker number.
+ *
+ * Leader callers pass a shared file handle and -1 for worker. After creating
+ * the tape set, use LogicalTapeImport() to import the worker tapes into it.
+ *
+ * Currently, the leader will only import worker tapes into the set, it does
+ * not create tapes of its own, although in principle that should work.
*/
LogicalTapeSet *
-LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
- SharedFileSet *fileset, int worker)
+LogicalTapeSetCreate(bool preallocate, SharedFileSet *fileset, int worker)
{
LogicalTapeSet *lts;
- int i;
/*
* Create top-level struct including per-tape LogicalTape structs.
*/
- Assert(ntapes > 0);
lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet));
lts->nBlocksAllocated = 0L;
lts->nBlocksWritten = 0L;
@@ -701,22 +578,21 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));
lts->nFreeBlocks = 0;
lts->enable_prealloc = preallocate;
- lts->nTapes = ntapes;
- lts->tapes = (LogicalTape *) palloc(ntapes * sizeof(LogicalTape));
- for (i = 0; i < ntapes; i++)
- ltsInitTape(<s->tapes[i]);
+ lts->fileset = fileset;
+ lts->worker = worker;
/*
* Create temp BufFile storage as required.
*
- * Leader concatenates worker tapes, which requires special adjustment to
- * final tapeset data. Things are simpler for the worker case and the
+ * In leader, we hijack the BufFile of the first tape that's imported, and
+ * concatenate the BufFiles of any subsequent tapes to that. Hence don't
+ * create a BufFile here. Things are simpler for the worker case and the
* serial case, though. They are generally very similar -- workers use a
* shared fileset, whereas serial sorts use a conventional serial BufFile.
*/
- if (shared)
- ltsConcatWorkerTapes(lts, shared, fileset);
+ if (fileset && worker == -1)
+ lts->pfile = NULL;
else if (fileset)
{
char filename[MAXPGPATH];
@@ -730,27 +606,147 @@ LogicalTapeSetCreate(int ntapes, bool preallocate, TapeShare *shared,
return lts;
}
+
/*
- * Close a logical tape set and release all resources.
+ * Claim ownership of a logical tape from an existing shared BufFile.
+ *
+ * Caller should be leader process. Though tapes are marked as frozen in
+ * workers, they are not frozen when opened within leader, since unfrozen tapes
+ * use a larger read buffer. (Frozen tapes have smaller read buffer, optimized
+ * for random access.)
*/
-void
-LogicalTapeSetClose(LogicalTapeSet *lts)
+LogicalTape *
+LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared)
{
LogicalTape *lt;
- int i;
+ long tapeblocks;
+ char filename[MAXPGPATH];
+ BufFile *file;
+ int64 filesize;
- BufFileClose(lts->pfile);
- for (i = 0; i < lts->nTapes; i++)
+ lt = ltsCreateTape(lts);
+
+ /*
+ * build concatenated view of all buffiles, remembering the block number
+ * where each source file begins.
+ */
+ pg_itoa(worker, filename);
+ file = BufFileOpenShared(lts->fileset, filename, O_RDONLY);
+ filesize = BufFileSize(file);
+
+ /*
+ * Stash first BufFile, and concatenate subsequent BufFiles to that. Store
+ * block offset into each tape as we go.
+ */
+ lt->firstBlockNumber = shared->firstblocknumber;
+ if (lts->pfile == NULL)
{
- lt = <s->tapes[i];
- if (lt->buffer)
- pfree(lt->buffer);
+ lts->pfile = file;
+ lt->offsetBlockNumber = 0L;
}
- pfree(lts->tapes);
+ else
+ {
+ lt->offsetBlockNumber = BufFileAppend(lts->pfile, file);
+ }
+ /* Don't allocate more for read buffer than could possibly help */
+ lt->max_size = Min(MaxAllocSize, filesize);
+ tapeblocks = filesize / BLCKSZ;
+
+ /*
+ * Update # of allocated blocks and # blocks written to reflect the
+ * imported BufFile. Allocated/written blocks include space used by holes
+ * left between concatenated BufFiles. Also track the number of hole
+ * blocks so that we can later work backwards to calculate the number of
+ * physical blocks for instrumentation.
+ */
+ lts->nHoleBlocks += lt->offsetBlockNumber - lts->nBlocksAllocated;
+
+ lts->nBlocksAllocated = lt->offsetBlockNumber + tapeblocks;
+ lts->nBlocksWritten = lts->nBlocksAllocated;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape set and release all resources.
+ *
+ * NOTE: This doesn't close any of the tapes! You must close them
+ * first, or you can let them be destroyed along with the memory context.
+ */
+void
+LogicalTapeSetClose(LogicalTapeSet *lts)
+{
+ BufFileClose(lts->pfile);
pfree(lts->freeBlocks);
pfree(lts);
}
+/*
+ * Create a logical tape in the given tapeset.
+ *
+ * The tape is initialized in write state.
+ */
+LogicalTape *
+LogicalTapeCreate(LogicalTapeSet *lts)
+{
+ /*
+ * The only thing that currently prevents creating new tapes in leader is
+ * the fact that BufFiles opened using BufFileOpenShared() are read-only
+ * by definition, but that could be changed if it seemed worthwhile. For
+ * now, writing to the leader tape will raise a "Bad file descriptor"
+ * error, so tuplesort must avoid writing to the leader tape altogether.
+ */
+ if (lts->fileset && lts->worker == -1)
+ elog(ERROR, "cannot create new tapes in leader process");
+
+ return ltsCreateTape(lts);
+}
+
+static LogicalTape *
+ltsCreateTape(LogicalTapeSet *lts)
+{
+ LogicalTape *lt;
+
+ /*
+ * Create per-tape struct. Note we allocate the I/O buffer lazily.
+ */
+ lt = palloc(sizeof(LogicalTape));
+ lt->tapeSet = lts;
+ lt->writing = true;
+ lt->frozen = false;
+ lt->dirty = false;
+ lt->firstBlockNumber = -1L;
+ lt->curBlockNumber = -1L;
+ lt->nextBlockNumber = -1L;
+ lt->offsetBlockNumber = 0L;
+ lt->buffer = NULL;
+ lt->buffer_size = 0;
+ /* palloc() larger than MaxAllocSize would fail */
+ lt->max_size = MaxAllocSize;
+ lt->pos = 0;
+ lt->nbytes = 0;
+ lt->prealloc = NULL;
+ lt->nprealloc = 0;
+ lt->prealloc_size = 0;
+
+ return lt;
+}
+
+/*
+ * Close a logical tape.
+ *
+ * Note: This doesn't return any blocks to the free list! You must read
+ * the tape to the end first, to reuse the space. In current use, though,
+ * we only close tapes after fully reading them.
+ */
+void
+LogicalTapeClose(LogicalTape *lt)
+{
+ if (lt->buffer)
+ pfree(lt->buffer);
+ pfree(lt);
+}
+
/*
* Mark a logical tape set as not needing management of free space anymore.
*
@@ -772,14 +768,11 @@ LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts)
* There are no error returns; we ereport() on failure.
*/
void
-LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -818,11 +811,11 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* First allocate the next block, so that we can store it in the
* 'next' pointer of this block.
*/
- nextBlockNumber = ltsGetBlock(lts, lt);
+ nextBlockNumber = ltsGetBlock(lt->tapeSet, lt);
/* set the next-pointer and dump the current block. */
TapeBlockGetTrailer(lt->buffer)->next = nextBlockNumber;
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
/* initialize the prev-pointer of the next block */
TapeBlockGetTrailer(lt->buffer)->prev = lt->curBlockNumber;
@@ -860,12 +853,9 @@ LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
* byte buffer is used.
*/
void
-LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
+LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
+ LogicalTapeSet *lts = lt->tapeSet;
/*
* Round and cap buffer_size if needed.
@@ -907,7 +897,7 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
}
@@ -939,61 +929,28 @@ LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum, size_t buffer_size)
}
}
-/*
- * Rewind logical tape and switch from reading to writing.
- *
- * NOTE: we assume the caller has read the tape to the end; otherwise
- * untouched data will not have been freed. We could add more code to free
- * any unread blocks, but in current usage of this module it'd be useless
- * code.
- */
-void
-LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum)
-{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
- Assert(!lt->writing && !lt->frozen);
- lt->writing = true;
- lt->dirty = false;
- lt->firstBlockNumber = -1L;
- lt->curBlockNumber = -1L;
- lt->pos = 0;
- lt->nbytes = 0;
- if (lt->buffer)
- pfree(lt->buffer);
- lt->buffer = NULL;
- lt->buffer_size = 0;
-}
-
/*
* Read from a logical tape.
*
* Early EOF is indicated by return value less than #bytes requested.
*/
size_t
-LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size)
+LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size)
{
- LogicalTape *lt;
size_t nread = 0;
size_t nthistime;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(!lt->writing);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
while (size > 0)
{
if (lt->pos >= lt->nbytes)
{
/* Try to load more data into buffer. */
- if (!ltsReadFillBuffer(lts, lt))
+ if (!ltsReadFillBuffer(lt))
break; /* EOF */
}
@@ -1031,12 +988,10 @@ LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
* Serial sorts should set share to NULL.
*/
void
-LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
+LogicalTapeFreeze(LogicalTape *lt, TapeShare *share)
{
- LogicalTape *lt;
+ LogicalTapeSet *lts = lt->tapeSet;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->writing);
Assert(lt->offsetBlockNumber == 0L);
@@ -1058,8 +1013,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
lt->buffer_size - lt->nbytes);
TapeBlockSetNBytes(lt->buffer, lt->nbytes);
- ltsWriteBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
- lt->writing = false;
+ ltsWriteBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
}
lt->writing = false;
lt->frozen = true;
@@ -1086,7 +1040,7 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
if (lt->firstBlockNumber == -1L)
lt->nextBlockNumber = -1L;
- ltsReadBlock(lts, lt->curBlockNumber, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, lt->curBlockNumber, (void *) lt->buffer);
if (TapeBlockIsLast(lt->buffer))
lt->nextBlockNumber = -1L;
else
@@ -1101,25 +1055,6 @@ LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum, TapeShare *share)
}
}
-/*
- * Add additional tapes to this tape set. Not intended to be used when any
- * tapes are frozen.
- */
-void
-LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
-{
- int i;
- int nTapesOrig = lts->nTapes;
-
- lts->nTapes += nAdditional;
-
- lts->tapes = (LogicalTape *) repalloc(lts->tapes,
- lts->nTapes * sizeof(LogicalTape));
-
- for (i = nTapesOrig; i < lts->nTapes; i++)
- ltsInitTape(<s->tapes[i]);
-}
-
/*
* Backspace the tape a given number of bytes. (We also support a more
* general seek interface, see below.)
@@ -1134,18 +1069,15 @@ LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional)
* that case.
*/
size_t
-LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
+LogicalTapeBackspace(LogicalTape *lt, size_t size)
{
- LogicalTape *lt;
size_t seekpos = 0;
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
/*
* Easy case for seek within current block.
@@ -1175,7 +1107,7 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
return seekpos;
}
- ltsReadBlock(lts, prev, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, prev, (void *) lt->buffer);
if (TapeBlockGetTrailer(lt->buffer)->next != lt->curBlockNumber)
elog(ERROR, "broken tape, next of block %ld is %ld, expected %ld",
@@ -1208,23 +1140,18 @@ LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum, size_t size)
* LogicalTapeTell().
*/
void
-LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset)
+LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
Assert(lt->frozen);
Assert(offset >= 0 && offset <= TapeBlockPayloadSize);
Assert(lt->buffer_size == BLCKSZ);
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
if (blocknum != lt->curBlockNumber)
{
- ltsReadBlock(lts, blocknum, (void *) lt->buffer);
+ ltsReadBlock(lt->tapeSet, blocknum, (void *) lt->buffer);
lt->curBlockNumber = blocknum;
lt->nbytes = TapeBlockPayloadSize;
lt->nextBlockNumber = TapeBlockGetTrailer(lt->buffer)->next;
@@ -1242,16 +1169,10 @@ LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
* the position for a seek after freezing. Not clear if anyone needs that.
*/
void
-LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset)
+LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset)
{
- LogicalTape *lt;
-
- Assert(tapenum >= 0 && tapenum < lts->nTapes);
- lt = <s->tapes[tapenum];
-
if (lt->buffer == NULL)
- ltsInitReadBuffer(lts, lt);
+ ltsInitReadBuffer(lt);
Assert(lt->offsetBlockNumber == 0L);
@@ -1271,12 +1192,5 @@ LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
long
LogicalTapeSetBlocks(LogicalTapeSet *lts)
{
-#ifdef USE_ASSERT_CHECKING
- for (int i = 0; i < lts->nTapes; i++)
- {
- LogicalTape *lt = <s->tapes[i];
- Assert(!lt->writing || lt->buffer == NULL);
- }
-#endif
return lts->nBlocksWritten - lts->nHoleBlocks;
}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb78..285f6127bc0 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -262,6 +262,7 @@ struct Tuplesortstate
MemoryContext sortcontext; /* memory context holding most sort data */
MemoryContext tuplecontext; /* sub-context of sortcontext for tuple data */
LogicalTapeSet *tapeset; /* logtape.c object for tapes in a temp file */
+ LogicalTape **tapes;
/*
* These function pointers decouple the routines that must know what kind
@@ -290,7 +291,7 @@ struct Tuplesortstate
* SortTuple struct!), and increase state->availMem by the amount of
* memory space thereby released.
*/
- void (*writetup) (Tuplesortstate *state, int tapenum,
+ void (*writetup) (Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
/*
@@ -299,7 +300,7 @@ struct Tuplesortstate
* from the slab memory arena, or is palloc'd, see readtup_alloc().
*/
void (*readtup) (Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
/*
* This array holds the tuples now in sort memory. If we are in state
@@ -393,7 +394,7 @@ struct Tuplesortstate
* the next tuple to return. (In the tape case, the tape's current read
* position is also critical state.)
*/
- int result_tape; /* actual tape number of finished output */
+ LogicalTape *result_tape; /* tape of finished output */
int current; /* array index (only used if SORTEDINMEM) */
bool eof_reached; /* reached EOF (needed for cursors) */
@@ -599,9 +600,9 @@ struct Sharedsort
*/
/* When using this macro, beware of double evaluation of len */
-#define LogicalTapeReadExact(tapeset, tapenum, ptr, len) \
+#define LogicalTapeReadExact(tape, ptr, len) \
do { \
- if (LogicalTapeRead(tapeset, tapenum, ptr, len) != (size_t) (len)) \
+ if (LogicalTapeRead(tape, ptr, len) != (size_t) (len)) \
elog(ERROR, "unexpected end of data"); \
} while(0)
@@ -619,7 +620,7 @@ static void init_slab_allocator(Tuplesortstate *state, int numSlots);
static void mergeruns(Tuplesortstate *state);
static void mergeonerun(Tuplesortstate *state);
static void beginmerge(Tuplesortstate *state);
-static bool mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup);
+static bool mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup);
static void dumptuples(Tuplesortstate *state, bool alltuples);
static void make_bounded_heap(Tuplesortstate *state);
static void sort_bounded_heap(Tuplesortstate *state);
@@ -628,39 +629,39 @@ static void tuplesort_heap_insert(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_replace_top(Tuplesortstate *state, SortTuple *tuple);
static void tuplesort_heap_delete_top(Tuplesortstate *state);
static void reversedirection(Tuplesortstate *state);
-static unsigned int getlen(Tuplesortstate *state, int tapenum, bool eofOK);
-static void markrunend(Tuplesortstate *state, int tapenum);
+static unsigned int getlen(LogicalTape *tape, bool eofOK);
+static void markrunend(LogicalTape *tape);
static void *readtup_alloc(Tuplesortstate *state, Size tuplen);
static int comparetup_heap(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_heap(Tuplesortstate *state, int tapenum,
+static void writetup_heap(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_cluster(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_cluster(Tuplesortstate *state, int tapenum,
+static void writetup_cluster(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_index_btree(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static int comparetup_index_hash(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_index(Tuplesortstate *state, int tapenum,
+static void writetup_index(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int comparetup_datum(const SortTuple *a, const SortTuple *b,
Tuplesortstate *state);
static void copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup);
-static void writetup_datum(Tuplesortstate *state, int tapenum,
+static void writetup_datum(Tuplesortstate *state, LogicalTape *tape,
SortTuple *stup);
static void readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len);
+ LogicalTape *tape, unsigned int len);
static int worker_get_identifier(Tuplesortstate *state);
static void worker_freeze_result_tape(Tuplesortstate *state);
static void worker_nomergeruns(Tuplesortstate *state);
@@ -869,7 +870,7 @@ tuplesort_begin_batch(Tuplesortstate *state)
* inittapes(), if needed
*/
- state->result_tape = -1; /* flag that result tape has not been formed */
+ state->result_tape = NULL; /* flag that result tape has not been formed */
MemoryContextSwitchTo(oldcontext);
}
@@ -2202,7 +2203,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
if (state->eof_reached)
return false;
- if ((tuplen = getlen(state, state->result_tape, true)) != 0)
+ if ((tuplen = getlen(state->result_tape, true)) != 0)
{
READTUP(state, stup, state->result_tape, tuplen);
@@ -2235,8 +2236,7 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* end of file; back up to fetch last tuple's ending length
* word. If seek fails we must have a completely empty file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
2 * sizeof(unsigned int));
if (nmoved == 0)
return false;
@@ -2250,20 +2250,18 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
* Back up and fetch previously-returned tuple's ending length
* word. If seek fails, assume we are at start of file.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
sizeof(unsigned int));
if (nmoved == 0)
return false;
else if (nmoved != sizeof(unsigned int))
elog(ERROR, "unexpected tape position");
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Back up to get ending length word of tuple before it.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen + 2 * sizeof(unsigned int));
if (nmoved == tuplen + sizeof(unsigned int))
{
@@ -2280,15 +2278,14 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
elog(ERROR, "bogus tuple length in backward scan");
}
- tuplen = getlen(state, state->result_tape, false);
+ tuplen = getlen(state->result_tape, false);
/*
* Now we have the length of the prior tuple, back up and read it.
* Note: READTUP expects we are positioned after the initial
* length word of the tuple, so back up to that point.
*/
- nmoved = LogicalTapeBackspace(state->tapeset,
- state->result_tape,
+ nmoved = LogicalTapeBackspace(state->result_tape,
tuplen);
if (nmoved != tuplen)
elog(ERROR, "bogus tuple length in backward scan");
@@ -2346,11 +2343,10 @@ tuplesort_gettuple_common(Tuplesortstate *state, bool forward,
tuplesort_heap_delete_top(state);
/*
- * Rewind to free the read buffer. It'd go away at the
- * end of the sort anyway, but better to release the
- * memory early.
+ * Close the tape. It'd go away at the end of the sort
+ * anyway, but better to release the memory early.
*/
- LogicalTapeRewindForWrite(state->tapeset, srcTape);
+ LogicalTapeClose(state->tapes[srcTape]);
return true;
}
newtup.srctape = srcTape;
@@ -2648,9 +2644,12 @@ inittapes(Tuplesortstate *state, bool mergeruns)
/* Create the tape set and allocate the per-tape data arrays */
inittapestate(state, maxTapes);
state->tapeset =
- LogicalTapeSetCreate(maxTapes, false, NULL,
+ LogicalTapeSetCreate(false,
state->shared ? &state->shared->fileset : NULL,
state->worker);
+ state->tapes = palloc(maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < maxTapes; j++)
+ state->tapes[j] = LogicalTapeCreate(state->tapeset);
state->currentRun = 0;
@@ -2900,7 +2899,7 @@ mergeruns(Tuplesortstate *state)
/* End of step D2: rewind all output tapes to prepare for merging */
for (tapenum = 0; tapenum < state->tapeRange; tapenum++)
- LogicalTapeRewindForRead(state->tapeset, tapenum, state->read_buffer_size);
+ LogicalTapeRewindForRead(state->tapes[tapenum], state->read_buffer_size);
for (;;)
{
@@ -2962,11 +2961,14 @@ mergeruns(Tuplesortstate *state)
/* Step D6: decrease level */
if (--state->Level == 0)
break;
+
/* rewind output tape T to use as new input */
- LogicalTapeRewindForRead(state->tapeset, state->tp_tapenum[state->tapeRange],
+ LogicalTapeRewindForRead(state->tapes[state->tp_tapenum[state->tapeRange]],
state->read_buffer_size);
- /* rewind used-up input tape P, and prepare it for write pass */
- LogicalTapeRewindForWrite(state->tapeset, state->tp_tapenum[state->tapeRange - 1]);
+
+ /* close used-up input tape P, and create a new one for write pass */
+ LogicalTapeClose(state->tapes[state->tp_tapenum[state->tapeRange - 1]]);
+ state->tapes[state->tp_tapenum[state->tapeRange - 1]] = LogicalTapeCreate(state->tapeset);
state->tp_runs[state->tapeRange - 1] = 0;
/*
@@ -2994,18 +2996,21 @@ mergeruns(Tuplesortstate *state)
* output tape while rewinding it. The last iteration of step D6 would be
* a waste of cycles anyway...
*/
- state->result_tape = state->tp_tapenum[state->tapeRange];
+ state->result_tape = state->tapes[state->tp_tapenum[state->tapeRange]];
if (!WORKER(state))
- LogicalTapeFreeze(state->tapeset, state->result_tape, NULL);
+ LogicalTapeFreeze(state->result_tape, NULL);
else
worker_freeze_result_tape(state);
state->status = TSS_SORTEDONTAPE;
- /* Release the read buffers of all the other tapes, by rewinding them. */
+ /* Close all the other tapes, to release their read buffers. */
for (tapenum = 0; tapenum < state->maxTapes; tapenum++)
{
- if (tapenum != state->result_tape)
- LogicalTapeRewindForWrite(state->tapeset, tapenum);
+ if (state->tapes[tapenum] != state->result_tape)
+ {
+ LogicalTapeClose(state->tapes[tapenum]);
+ state->tapes[tapenum] = NULL;
+ }
}
}
@@ -3018,7 +3023,8 @@ mergeruns(Tuplesortstate *state)
static void
mergeonerun(Tuplesortstate *state)
{
- int destTape = state->tp_tapenum[state->tapeRange];
+ int destTapeNum = state->tp_tapenum[state->tapeRange];
+ LogicalTape *destTape = state->tapes[destTapeNum];
int srcTape;
/*
@@ -3061,7 +3067,7 @@ mergeonerun(Tuplesortstate *state)
* When the heap empties, we're done. Write an end-of-run marker on the
* output tape, and increment its count of real runs.
*/
- markrunend(state, destTape);
+ markrunend(destTape);
state->tp_runs[state->tapeRange]++;
#ifdef TRACE_SORT
@@ -3127,17 +3133,18 @@ beginmerge(Tuplesortstate *state)
* Returns false on EOF.
*/
static bool
-mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
+mergereadnext(Tuplesortstate *state, int srcTapeIndex, SortTuple *stup)
{
+ LogicalTape *srcTape = state->tapes[srcTapeIndex];
unsigned int tuplen;
- if (!state->mergeactive[srcTape])
+ if (!state->mergeactive[srcTapeIndex])
return false; /* tape's run is already exhausted */
/* read next tuple, if any */
- if ((tuplen = getlen(state, srcTape, true)) == 0)
+ if ((tuplen = getlen(srcTape, true)) == 0)
{
- state->mergeactive[srcTape] = false;
+ state->mergeactive[srcTapeIndex] = false;
return false;
}
READTUP(state, stup, srcTape, tuplen);
@@ -3154,6 +3161,7 @@ mergereadnext(Tuplesortstate *state, int srcTape, SortTuple *stup)
static void
dumptuples(Tuplesortstate *state, bool alltuples)
{
+ LogicalTape *destTape;
int memtupwrite;
int i;
@@ -3220,10 +3228,10 @@ dumptuples(Tuplesortstate *state, bool alltuples)
#endif
memtupwrite = state->memtupcount;
+ destTape = state->tapes[state->tp_tapenum[state->destTape]];
for (i = 0; i < memtupwrite; i++)
{
- WRITETUP(state, state->tp_tapenum[state->destTape],
- &state->memtuples[i]);
+ WRITETUP(state, destTape, &state->memtuples[i]);
state->memtupcount--;
}
@@ -3236,7 +3244,7 @@ dumptuples(Tuplesortstate *state, bool alltuples)
*/
MemoryContextReset(state->tuplecontext);
- markrunend(state, state->tp_tapenum[state->destTape]);
+ markrunend(destTape);
state->tp_runs[state->destTape]++;
state->tp_dummy[state->destTape]--; /* per Alg D step D2 */
@@ -3270,9 +3278,7 @@ tuplesort_rescan(Tuplesortstate *state)
state->markpos_eof = false;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeRewindForRead(state->tapeset,
- state->result_tape,
- 0);
+ LogicalTapeRewindForRead(state->result_tape, 0);
state->eof_reached = false;
state->markpos_block = 0L;
state->markpos_offset = 0;
@@ -3303,8 +3309,7 @@ tuplesort_markpos(Tuplesortstate *state)
state->markpos_eof = state->eof_reached;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeTell(state->tapeset,
- state->result_tape,
+ LogicalTapeTell(state->result_tape,
&state->markpos_block,
&state->markpos_offset);
state->markpos_eof = state->eof_reached;
@@ -3335,8 +3340,7 @@ tuplesort_restorepos(Tuplesortstate *state)
state->eof_reached = state->markpos_eof;
break;
case TSS_SORTEDONTAPE:
- LogicalTapeSeek(state->tapeset,
- state->result_tape,
+ LogicalTapeSeek(state->result_tape,
state->markpos_block,
state->markpos_offset);
state->eof_reached = state->markpos_eof;
@@ -3678,11 +3682,11 @@ reversedirection(Tuplesortstate *state)
*/
static unsigned int
-getlen(Tuplesortstate *state, int tapenum, bool eofOK)
+getlen(LogicalTape *tape, bool eofOK)
{
unsigned int len;
- if (LogicalTapeRead(state->tapeset, tapenum,
+ if (LogicalTapeRead(tape,
&len, sizeof(len)) != sizeof(len))
elog(ERROR, "unexpected end of tape");
if (len == 0 && !eofOK)
@@ -3691,11 +3695,11 @@ getlen(Tuplesortstate *state, int tapenum, bool eofOK)
}
static void
-markrunend(Tuplesortstate *state, int tapenum)
+markrunend(LogicalTape *tape)
{
unsigned int len = 0;
- LogicalTapeWrite(state->tapeset, tapenum, (void *) &len, sizeof(len));
+ LogicalTapeWrite(tape, (void *) &len, sizeof(len));
}
/*
@@ -3873,7 +3877,7 @@ copytup_heap(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_heap(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
MinimalTuple tuple = (MinimalTuple) stup->tuple;
@@ -3884,13 +3888,10 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
/* total on-disk footprint: */
unsigned int tuplen = tupbodylen + sizeof(int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tupbody, tupbodylen);
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -3901,7 +3902,7 @@ writetup_heap(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_heap(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tupbodylen = len - sizeof(int);
unsigned int tuplen = tupbodylen + MINIMAL_TUPLE_DATA_OFFSET;
@@ -3911,11 +3912,9 @@ readtup_heap(Tuplesortstate *state, SortTuple *stup,
/* read in the tuple proper */
tuple->t_len = tuplen;
- LogicalTapeReadExact(state->tapeset, tapenum,
- tupbody, tupbodylen);
+ LogicalTapeReadExact(tape, tupbody, tupbodylen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
htup.t_len = tuple->t_len + MINIMAL_TUPLE_OFFSET;
@@ -4116,21 +4115,17 @@ copytup_cluster(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_cluster(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
HeapTuple tuple = (HeapTuple) stup->tuple;
unsigned int tuplen = tuple->t_len + sizeof(ItemPointerData) + sizeof(int);
/* We need to store t_self, but not other fields of HeapTupleData */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
- LogicalTapeWrite(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeWrite(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4141,7 +4136,7 @@ writetup_cluster(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_cluster(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int tuplen)
+ LogicalTape *tape, unsigned int tuplen)
{
unsigned int t_len = tuplen - sizeof(ItemPointerData) - sizeof(int);
HeapTuple tuple = (HeapTuple) readtup_alloc(state,
@@ -4150,16 +4145,13 @@ readtup_cluster(Tuplesortstate *state, SortTuple *stup,
/* Reconstruct the HeapTupleData header */
tuple->t_data = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
tuple->t_len = t_len;
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuple->t_self, sizeof(ItemPointerData));
+ LogicalTapeReadExact(tape, &tuple->t_self, sizeof(ItemPointerData));
/* We don't currently bother to reconstruct t_tableOid */
tuple->t_tableOid = InvalidOid;
/* Read in the tuple body */
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple->t_data, tuple->t_len);
+ LogicalTapeReadExact(tape, tuple->t_data, tuple->t_len);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value, if it's a simple column */
if (state->indexInfo->ii_IndexAttrNumbers[0] != 0)
@@ -4373,19 +4365,16 @@ copytup_index(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_index(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
IndexTuple tuple = (IndexTuple) stup->tuple;
unsigned int tuplen;
tuplen = IndexTupleSize(tuple) + sizeof(tuplen);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) tuple, IndexTupleSize(tuple));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) tuple, IndexTupleSize(tuple));
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &tuplen, sizeof(tuplen));
+ LogicalTapeWrite(tape, (void *) &tuplen, sizeof(tuplen));
if (!state->slabAllocatorUsed)
{
@@ -4396,16 +4385,14 @@ writetup_index(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_index(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
IndexTuple tuple = (IndexTuple) readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- tuple, tuplen);
+ LogicalTapeReadExact(tape, tuple, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
stup->tuple = (void *) tuple;
/* set up first-column key value */
stup->datum1 = index_getattr(tuple,
@@ -4447,7 +4434,7 @@ copytup_datum(Tuplesortstate *state, SortTuple *stup, void *tup)
}
static void
-writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
+writetup_datum(Tuplesortstate *state, LogicalTape *tape, SortTuple *stup)
{
void *waddr;
unsigned int tuplen;
@@ -4472,13 +4459,10 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
writtenlen = tuplen + sizeof(unsigned int);
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
- LogicalTapeWrite(state->tapeset, tapenum,
- waddr, tuplen);
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, waddr, tuplen);
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeWrite(state->tapeset, tapenum,
- (void *) &writtenlen, sizeof(writtenlen));
+ LogicalTapeWrite(tape, (void *) &writtenlen, sizeof(writtenlen));
if (!state->slabAllocatorUsed && stup->tuple)
{
@@ -4489,7 +4473,7 @@ writetup_datum(Tuplesortstate *state, int tapenum, SortTuple *stup)
static void
readtup_datum(Tuplesortstate *state, SortTuple *stup,
- int tapenum, unsigned int len)
+ LogicalTape *tape, unsigned int len)
{
unsigned int tuplen = len - sizeof(unsigned int);
@@ -4503,8 +4487,7 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
else if (!state->tuples)
{
Assert(tuplen == sizeof(Datum));
- LogicalTapeReadExact(state->tapeset, tapenum,
- &stup->datum1, tuplen);
+ LogicalTapeReadExact(tape, &stup->datum1, tuplen);
stup->isnull1 = false;
stup->tuple = NULL;
}
@@ -4512,16 +4495,14 @@ readtup_datum(Tuplesortstate *state, SortTuple *stup,
{
void *raddr = readtup_alloc(state, tuplen);
- LogicalTapeReadExact(state->tapeset, tapenum,
- raddr, tuplen);
+ LogicalTapeReadExact(tape, raddr, tuplen);
stup->datum1 = PointerGetDatum(raddr);
stup->isnull1 = false;
stup->tuple = raddr;
}
if (state->randomAccess) /* need trailing length word? */
- LogicalTapeReadExact(state->tapeset, tapenum,
- &tuplen, sizeof(tuplen));
+ LogicalTapeReadExact(tape, &tuplen, sizeof(tuplen));
}
/*
@@ -4633,7 +4614,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
TapeShare output;
Assert(WORKER(state));
- Assert(state->result_tape != -1);
+ Assert(state->result_tape != NULL);
Assert(state->memtupcount == 0);
/*
@@ -4649,7 +4630,7 @@ worker_freeze_result_tape(Tuplesortstate *state)
* Parallel worker requires result tape metadata, which is to be stored in
* shared memory for leader
*/
- LogicalTapeFreeze(state->tapeset, state->result_tape, &output);
+ LogicalTapeFreeze(state->result_tape, &output);
/* Store properties of output tape, and update finished worker count */
SpinLockAcquire(&shared->mutex);
@@ -4668,9 +4649,9 @@ static void
worker_nomergeruns(Tuplesortstate *state)
{
Assert(WORKER(state));
- Assert(state->result_tape == -1);
+ Assert(state->result_tape == NULL);
- state->result_tape = state->tp_tapenum[state->destTape];
+ state->result_tape = state->tapes[state->tp_tapenum[state->destTape]];
worker_freeze_result_tape(state);
}
@@ -4714,9 +4695,13 @@ leader_takeover_tapes(Tuplesortstate *state)
* randomAccess is disallowed for parallel sorts.
*/
inittapestate(state, nParticipants + 1);
- state->tapeset = LogicalTapeSetCreate(nParticipants + 1, false,
- shared->tapes, &shared->fileset,
+ state->tapeset = LogicalTapeSetCreate(false,
+ &shared->fileset,
state->worker);
+ state->tapes = palloc(state->maxTapes * sizeof(LogicalTape *));
+ for (j = 0; j < nParticipants; j++)
+ state->tapes[j] = LogicalTapeImport(state->tapeset, j, &shared->tapes[j]);
+ /* tapes[nParticipants] represents the "leader tape", which is not used */
/* mergeruns() relies on currentRun for # of runs (in one-pass cases) */
state->currentRun = nParticipants;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index d65099c94aa..da7df7fd5f2 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -40,6 +40,7 @@ struct ExprContext;
struct RangeTblEntry; /* avoid including parsenodes.h here */
struct ExprEvalStep; /* avoid including execExpr.h everywhere */
struct CopyMultiInsertBuffer;
+struct LogicalTapeSet;
/* ----------------
@@ -2178,7 +2179,7 @@ typedef struct AggState
bool table_filled; /* hash table filled yet? */
int num_hashes;
MemoryContext hash_metacxt; /* memory for hash table itself */
- struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
+ struct LogicalTapeSet *hash_tapeset; /* tape set for hash spill tapes */
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
* exists only during first pass */
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
diff --git a/src/include/utils/logtape.h b/src/include/utils/logtape.h
index 85d2e03c631..758a19779c6 100644
--- a/src/include/utils/logtape.h
+++ b/src/include/utils/logtape.h
@@ -18,9 +18,13 @@
#include "storage/sharedfileset.h"
-/* LogicalTapeSet is an opaque type whose details are not known outside logtape.c. */
-
+/*
+ * LogicalTapeSet and LogicalTape are opaque types whose details are not
+ * known outside logtape.c.
+ */
typedef struct LogicalTapeSet LogicalTapeSet;
+typedef struct LogicalTape LogicalTape;
+
/*
* The approach tuplesort.c takes to parallel external sorts is that workers,
@@ -54,27 +58,20 @@ typedef struct TapeShare
* prototypes for functions in logtape.c
*/
-extern LogicalTapeSet *LogicalTapeSetCreate(int ntapes, bool preallocate,
- TapeShare *shared,
+extern LogicalTapeSet *LogicalTapeSetCreate(bool preallocate,
SharedFileSet *fileset, int worker);
+extern void LogicalTapeClose(LogicalTape *lt);
extern void LogicalTapeSetClose(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeCreate(LogicalTapeSet *lts);
+extern LogicalTape *LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared);
extern void LogicalTapeSetForgetFreeSpace(LogicalTapeSet *lts);
-extern size_t LogicalTapeRead(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeWrite(LogicalTapeSet *lts, int tapenum,
- void *ptr, size_t size);
-extern void LogicalTapeRewindForRead(LogicalTapeSet *lts, int tapenum,
- size_t buffer_size);
-extern void LogicalTapeRewindForWrite(LogicalTapeSet *lts, int tapenum);
-extern void LogicalTapeFreeze(LogicalTapeSet *lts, int tapenum,
- TapeShare *share);
-extern void LogicalTapeSetExtend(LogicalTapeSet *lts, int nAdditional);
-extern size_t LogicalTapeBackspace(LogicalTapeSet *lts, int tapenum,
- size_t size);
-extern void LogicalTapeSeek(LogicalTapeSet *lts, int tapenum,
- long blocknum, int offset);
-extern void LogicalTapeTell(LogicalTapeSet *lts, int tapenum,
- long *blocknum, int *offset);
+extern size_t LogicalTapeRead(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeWrite(LogicalTape *lt, void *ptr, size_t size);
+extern void LogicalTapeRewindForRead(LogicalTape *lt, size_t buffer_size);
+extern void LogicalTapeFreeze(LogicalTape *lt, TapeShare *share);
+extern size_t LogicalTapeBackspace(LogicalTape *lt, size_t size);
+extern void LogicalTapeSeek(LogicalTape *lt, long blocknum, int offset);
+extern void LogicalTapeTell(LogicalTape *lt, long *blocknum, int *offset);
extern long LogicalTapeSetBlocks(LogicalTapeSet *lts);
#endif /* LOGTAPE_H */
--
2.29.2
--------------631E0A48DCAEA386819DFCEC
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0002-Replace-polyphase-merge-algorithm-with-a-simple-b.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Test 002_pg_upgrade fails with olddump on Windows
@ 2023-12-06 08:00 Alexander Lakhin <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Alexander Lakhin @ 2023-12-06 08:00 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: pgsql-hackers
06.12.2023 04:17, Michael Paquier wrote:
> At the end, just applying the filtering all the time makes the most
> sense to me, so I've applied a patch doing just that.
Thank you for the fix!
Now that test with the minimal dump passes fine, but when I tried to run
it with a complete dump borrowed from a normal test run:
set olddump=& set oldinstall=& set PG_TEST_NOCLEAN=1& meson test pg_upgrade/002_pg_upgrade
REM this test succeeded
copy testrun\pg_upgrade\002_pg_upgrade\data\tmp_test_*\dump1.sql
set olddump=c:\temp\dump1.sql& set oldinstall=%CD%/tmp_install/usr/local/pgsql& meson test pg_upgrade/002_pg_upgrade
I encountered another failure:
...
Creating dump of database schemas ok
Checking for presence of required libraries fatal
Your installation references loadable libraries that are missing from the
new installation. You can add these libraries to the new installation,
or remove the functions using them from the old installation. A list of
problem libraries is in the file:
.../build/testrun/pg_upgrade/002_pg_upgrade/data/t_002_pg_upgrade_new_node_data/pgdata/pg_upgrade_output.d/20231205T223247.304/loadable_libraries.txt
Failure, exiting
[22:32:51.086](3.796s) not ok 11 - run of pg_upgrade for new instance
...
loadable_libraries.txt contains:
could not load library ".../src/test/regress/refint.dll": ERROR: could not access file
".../src/test/regress/refint.dll": No such file or directory
In database: regression
could not load library ".../src/test/regress/autoinc.dll": ERROR: could not access file
".../src/test/regress/autoinc.dll": No such file or directory
In database: regression
could not load library ".../src/test/regress/regress.dll": ERROR: could not access file
".../src/test/regress/regress.dll": No such file or directory
In database: regression
Really, I can see refint.dll in ...\build\src\test\regress and in
...\build\tmp_install\usr\local\pgsql\lib, but not in
.../src/test/regress/regress.dll
c:\temp\dump1.sql contains:
...
CREATE FUNCTION public.check_primary_key() RETURNS trigger
LANGUAGE c
AS '.../build/src/test/regress/refint.dll', 'check_primary_key';
while ...\build\testrun\pg_upgrade\002_pg_upgrade\data\tmp_test_T6jE\dump1.sql
(for the failed test):
...
CREATE FUNCTION public.check_primary_key() RETURNS trigger
LANGUAGE c
AS '.../src/test/regress/refint.dll', 'check_primary_key';
The same is on Linux:
PG_TEST_NOCLEAN=1 meson test pg_upgrade/002_pg_upgrade
cp testrun/pg_upgrade/002_pg_upgrade/data/tmp_test_*/dump1.sql /tmp/
olddump=/tmp/dump1.sql oldinstall=`pwd`/tmp_install/usr/local/pgsql meson test pg_upgrade/002_pg_upgrade
So it looks like
my $newregresssrc = "$srcdir/src/test/regress";
is incorrect for meson.
Maybe it should be?:
my $newregresssrc = dirname($ENV{REGRESS_SHLIB});
(With this line the test passes for me on Windows and Linux).
Best regards,
Alexander
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Test 002_pg_upgrade fails with olddump on Windows
@ 2023-12-07 08:44 Michael Paquier <[email protected]>
parent: Alexander Lakhin <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Michael Paquier @ 2023-12-07 08:44 UTC (permalink / raw)
To: Alexander Lakhin <[email protected]>; +Cc: pgsql-hackers
On Wed, Dec 06, 2023 at 11:00:01AM +0300, Alexander Lakhin wrote:
> So it looks like
> my $newregresssrc = "$srcdir/src/test/regress";
> is incorrect for meson.
> Maybe it should be?:
> my $newregresssrc = dirname($ENV{REGRESS_SHLIB});
> (With this line the test passes for me on Windows and Linux).
Hmm. Yes, it looks like you're right here. That should allow all the
scenarios we expect to work to update the paths for the functions.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 27+ messages in thread
end of thread, other threads:[~2023-12-07 08:44 UTC | newest]
Thread overview: 27+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2017-10-26 03:29 Re: proposal: Support Unicode host variable in ECPG Jing Wang <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v2 1/2] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v5 1/2] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH 1/2] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2020-10-21 18:57 [PATCH v3 1/3] Refactor LogicalTapeSet/LogicalTape interface. Heikki Linnakangas <[email protected]>
2023-12-06 08:00 Re: Test 002_pg_upgrade fails with olddump on Windows Alexander Lakhin <[email protected]>
2023-12-07 08:44 ` Re: Test 002_pg_upgrade fails with olddump on Windows Michael Paquier <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox