pgpool2.git / summary / log / commit / refs
commit 69972bb530764ee57cb3d1eade7b839645a7d3f4
Author: Tatsuo Ishii <ishii at sraoss.co.jp>
Date: Thu Dec 23 00:57:02 2010 +0000
Import PostgreSQL 9.0 parser. This allows to use CREATE INDEX with
implicit index name, which is new in 9.0.
Allow to use set standard_conforming_string command. It affetcs the
scanner in PostgreSQL 9.0 or later.
Change outfuncs.c to adopt to the effect of server encoding. This
happens in PostgreSQL 9.0 parser.
Patch provided by Akio Ishida.
parser/pg_wchar.h | 469 +++++++++++++++
parser/scanner.h | 119 ++++
parser/scansup.c | 220 +++++++
parser/wchar.c | 1640 +++++++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 2448 insertions(+)
diff --git a/parser/pg_wchar.h b/parser/pg_wchar.h
new file mode 100644
index 000000000..817f9aaaa
--- /dev/null
+++ b/parser/pg_wchar.h
@@ -0,0 +1,469 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_wchar.h
+ * multibyte-character support
+ *
+ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * $PostgreSQL: pgsql/src/include/mb/pg_wchar.h,v 1.94 2010/02/26 02:01:25 momjian Exp $
+ *
+ * NOTES
+ * This is used both by the backend and by libpq, but should not be
+ * included by libpq client programs. In particular, a libpq client
+ * should not assume that the encoding IDs used by the version of libpq
+ * it's linked to match up with the IDs declared here.
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_WCHAR_H
+#define PG_WCHAR_H
+
+#include <sys/types.h>
+
+/*
+ * The pg_wchar type
+ */
+typedef unsigned int pg_wchar;
+
+/*
+ * various definitions for EUC
+ */
+#define SS2 0x8e /* single shift 2 (JIS0201) */
+#define SS3 0x8f /* single shift 3 (JIS0212) */
+
+/*
+ * SJIS validation macros
+ */
+#define ISSJISHEAD(c) (((c) >= 0x81 && (c) <= 0x9f) || ((c) >= 0xe0 && (c) <= 0xfc))
+#define ISSJISTAIL(c) (((c) >= 0x40 && (c) <= 0x7e) || ((c) >= 0x80 && (c) <= 0xfc))
+
+/*
+ * Leading byte types or leading prefix byte for MULE internal code.
+ * See http://www.xemacs.org for more details. (there is a doc titled
+ * "XEmacs Internals Manual", "MULE Character Sets and Encodings"
+ * section.)
+ */
+/*
+ * Is a leading byte for "official" single byte encodings?
+ */
+#define IS_LC1(c) ((unsigned char)(c) >= 0x81 && (unsigned char)(c) <= 0x8d)
+/*
+ * Is a prefix byte for "private" single byte encodings?
+ */
+#define IS_LCPRV1(c) ((unsigned char)(c) == 0x9a || (unsigned char)(c) == 0x9b)
+/*
+ * Is a leading byte for "official" multibyte encodings?
+ */
+#define IS_LC2(c) ((unsigned char)(c) >= 0x90 && (unsigned char)(c) <= 0x99)
+/*
+ * Is a prefix byte for "private" multibyte encodings?
+ */
+#define IS_LCPRV2(c) ((unsigned char)(c) == 0x9c || (unsigned char)(c) == 0x9d)
+
+/*----------------------------------------------------
+ * leading characters
+ *----------------------------------------------------
+ */
+
+/*
+ * Official single byte encodings (0x81-0x8e)
+ */
+#define LC_ISO8859_1 0x81 /* ISO8859 Latin 1 */
+#define LC_ISO8859_2 0x82 /* ISO8859 Latin 2 */
+#define LC_ISO8859_3 0x83 /* ISO8859 Latin 3 */
+#define LC_ISO8859_4 0x84 /* ISO8859 Latin 4 */
+#define LC_TIS620 0x85 /* Thai (not supported yet) */
+#define LC_ISO8859_7 0x86 /* Greek (not supported yet) */
+#define LC_ISO8859_6 0x87 /* Arabic (not supported yet) */
+#define LC_ISO8859_8 0x88 /* Hebrew (not supported yet) */
+#define LC_JISX0201K 0x89 /* Japanese 1 byte kana */
+#define LC_JISX0201R 0x8a /* Japanese 1 byte Roman */
+/* Note that 0x8b seems to be unused as of Emacs 20.7.
+ * However, there might be a chance that 0x8b could be used
+ * in later version of Emacs.
+ */
+#define LC_KOI8_R 0x8b /* Cyrillic KOI8-R */
+#define LC_KOI8_U 0x8b /* Cyrillic KOI8-U */
+#define LC_ISO8859_5 0x8c /* ISO8859 Cyrillic */
+#define LC_ISO8859_9 0x8d /* ISO8859 Latin 5 (not supported yet) */
+/* #define FREE 0x8e free (unused) */
+
+/*
+ * Unused
+ */
+#define CONTROL_1 0x8f /* control characters (unused) */
+
+/*
+ * Official multibyte byte encodings (0x90-0x99)
+ * 0x9a-0x9d are free. 0x9e and 0x9f are reserved.
+ */
+#define LC_JISX0208_1978 0x90 /* Japanese Kanji, old JIS (not supported) */
+/* #define FREE 0x90 free (unused) */
+#define LC_GB2312_80 0x91 /* Chinese */
+#define LC_JISX0208 0x92 /* Japanese Kanji (JIS X 0208) */
+#define LC_KS5601 0x93 /* Korean */
+#define LC_JISX0212 0x94 /* Japanese Kanji (JIS X 0212) */
+#define LC_CNS11643_1 0x95 /* CNS 11643-1992 Plane 1 */
+#define LC_CNS11643_2 0x96 /* CNS 11643-1992 Plane 2 */
+/* #define FREE 0x97 free (unused) */
+#define LC_BIG5_1 0x98 /* Plane 1 Chinese traditional (not supported) */
+#define LC_BIG5_2 0x99 /* Plane 1 Chinese traditional (not supported) */
+
+/*
+ * Private single byte encodings (0xa0-0xef)
+ */
+#define LC_SISHENG 0xa0/* Chinese SiSheng characters for
+ * PinYin/ZhuYin (not supported) */
+#define LC_IPA 0xa1/* IPA (International Phonetic Association)
+ * (not supported) */
+#define LC_VISCII_LOWER 0xa2/* Vietnamese VISCII1.1 lower-case (not
+ * supported) */
+#define LC_VISCII_UPPER 0xa3/* Vietnamese VISCII1.1 upper-case (not
+ * supported) */
+#define LC_ARABIC_DIGIT 0xa4 /* Arabic digit (not supported) */
+#define LC_ARABIC_1_COLUMN 0xa5 /* Arabic 1-column (not supported) */
+#define LC_ASCII_RIGHT_TO_LEFT 0xa6 /* ASCII (left half of ISO8859-1) with
+ * right-to-left direction (not
+ * supported) */
+#define LC_LAO 0xa7/* Lao characters (ISO10646 0E80..0EDF) (not
+ * supported) */
+#define LC_ARABIC_2_COLUMN 0xa8 /* Arabic 1-column (not supported) */
+
+/*
+ * Private multibyte encodings (0xf0-0xff)
+ */
+#define LC_INDIAN_1_COLUMN 0xf0/* Indian charset for 1-column width glypps
+ * (not supported) */
+#define LC_TIBETAN_1_COLUMN 0xf1 /* Tibetan 1 column glyph (not supported) */
+#define LC_ETHIOPIC 0xf5 /* Ethiopic characters (not supported) */
+#define LC_CNS11643_3 0xf6 /* CNS 11643-1992 Plane 3 */
+#define LC_CNS11643_4 0xf7 /* CNS 11643-1992 Plane 4 */
+#define LC_CNS11643_5 0xf8 /* CNS 11643-1992 Plane 5 */
+#define LC_CNS11643_6 0xf9 /* CNS 11643-1992 Plane 6 */
+#define LC_CNS11643_7 0xfa /* CNS 11643-1992 Plane 7 */
+#define LC_INDIAN_2_COLUMN 0xfb/* Indian charset for 2-column width glypps
+ * (not supported) */
+#define LC_TIBETAN 0xfc /* Tibetan (not supported) */
+/* #define FREE 0xfd free (unused) */
+/* #define FREE 0xfe free (unused) */
+/* #define FREE 0xff free (unused) */
+
+/*
+ * PostgreSQL encoding identifiers
+ *
+ * WARNING: the order of this enum must be same as order of entries
+ * in the pg_enc2name_tbl[] array (in mb/encnames.c), and
+ * in the pg_wchar_table[] array (in mb/wchar.c)!
+ *
+ * If you add some encoding don't forget to check
+ * PG_ENCODING_BE_LAST macro.
+ *
+ * PG_SQL_ASCII is default encoding and must be = 0.
+ *
+ * XXX We must avoid renumbering any backend encoding until libpq's major
+ * version number is increased beyond 5; it turns out that the backend
+ * encoding IDs are effectively part of libpq's ABI as far as 8.2 initdb and
+ * psql are concerned.
+ */
+typedef enum pg_enc
+{
+ PG_SQL_ASCII = 0, /* SQL/ASCII */
+ PG_EUC_JP, /* EUC for Japanese */
+ PG_EUC_CN, /* EUC for Chinese */
+ PG_EUC_KR, /* EUC for Korean */
+ PG_EUC_TW, /* EUC for Taiwan */
+ PG_EUC_JIS_2004, /* EUC-JIS-2004 */
+ PG_UTF8, /* Unicode UTF8 */
+ PG_MULE_INTERNAL, /* Mule internal code */
+ PG_LATIN1, /* ISO-8859-1 Latin 1 */
+ PG_LATIN2, /* ISO-8859-2 Latin 2 */
+ PG_LATIN3, /* ISO-8859-3 Latin 3 */
+ PG_LATIN4, /* ISO-8859-4 Latin 4 */
+ PG_LATIN5, /* ISO-8859-9 Latin 5 */
+ PG_LATIN6, /* ISO-8859-10 Latin6 */
+ PG_LATIN7, /* ISO-8859-13 Latin7 */
+ PG_LATIN8, /* ISO-8859-14 Latin8 */
+ PG_LATIN9, /* ISO-8859-15 Latin9 */
+ PG_LATIN10, /* ISO-8859-16 Latin10 */
+ PG_WIN1256, /* windows-1256 */
+ PG_WIN1258, /* Windows-1258 */
+ PG_WIN866, /* (MS-DOS CP866) */
+ PG_WIN874, /* windows-874 */
+ PG_KOI8R, /* KOI8-R */
+ PG_WIN1251, /* windows-1251 */
+ PG_WIN1252, /* windows-1252 */
+ PG_ISO_8859_5, /* ISO-8859-5 */
+ PG_ISO_8859_6, /* ISO-8859-6 */
+ PG_ISO_8859_7, /* ISO-8859-7 */
+ PG_ISO_8859_8, /* ISO-8859-8 */
+ PG_WIN1250, /* windows-1250 */
+ PG_WIN1253, /* windows-1253 */
+ PG_WIN1254, /* windows-1254 */
+ PG_WIN1255, /* windows-1255 */
+ PG_WIN1257, /* windows-1257 */
+ PG_KOI8U, /* KOI8-U */
+ /* PG_ENCODING_BE_LAST points to the above entry */
+
+ /* followings are for client encoding only */
+ PG_SJIS, /* Shift JIS (Winindows-932) */
+ PG_BIG5, /* Big5 (Windows-950) */
+ PG_GBK, /* GBK (Windows-936) */
+ PG_UHC, /* UHC (Windows-949) */
+ PG_GB18030, /* GB18030 */
+ PG_JOHAB, /* EUC for Korean JOHAB */
+ PG_SHIFT_JIS_2004, /* Shift-JIS-2004 */
+ _PG_LAST_ENCODING_ /* mark only */
+
+} pg_enc;
+
+#define PG_ENCODING_BE_LAST PG_KOI8U
+
+/*
+ * Please use these tests before access to pg_encconv_tbl[]
+ * or to other places...
+ */
+#define PG_VALID_BE_ENCODING(_enc) \
+ ((_enc) >= 0 && (_enc) <= PG_ENCODING_BE_LAST)
+
+#define PG_ENCODING_IS_CLIENT_ONLY(_enc) \
+ ((_enc) > PG_ENCODING_BE_LAST && (_enc) < _PG_LAST_ENCODING_)
+
+#define PG_VALID_ENCODING(_enc) \
+ ((_enc) >= 0 && (_enc) < _PG_LAST_ENCODING_)
+
+/* On FE are possible all encodings */
+#define PG_VALID_FE_ENCODING(_enc) PG_VALID_ENCODING(_enc)
+
+/*
+ * Encoding names with all aliases
+ */
+typedef struct pg_encname
+{
+ char *name;
+ pg_enc encoding;
+} pg_encname;
+
+extern pg_encname pg_encname_tbl[];
+extern unsigned int pg_encname_tbl_sz;
+
+/*
+ * Careful:
+ *
+ * if (PG_VALID_ENCODING(encoding))
+ * pg_enc2name_tbl[ encoding ];
+ */
+typedef struct pg_enc2name
+{
+ char *name;
+ pg_enc encoding;
+#ifdef WIN32
+ unsigned codepage; /* codepage for WIN32 */
+#endif
+} pg_enc2name;
+
+extern pg_enc2name pg_enc2name_tbl[];
+
+/*
+ * Encoding names for gettext
+ */
+typedef struct pg_enc2gettext
+{
+ pg_enc encoding;
+ const char *name;
+} pg_enc2gettext;
+
+extern pg_enc2gettext pg_enc2gettext_tbl[];
+
+/*
+ * pg_wchar stuff
+ */
+typedef int (*mb2wchar_with_len_converter) (const unsigned char *from,
+ pg_wchar *to,
+ int len);
+
+typedef int (*mblen_converter) (const unsigned char *mbstr);
+
+typedef int (*mbdisplaylen_converter) (const unsigned char *mbstr);
+
+typedef int (*mbverifier) (const unsigned char *mbstr, int len);
+
+typedef struct
+{
+ mb2wchar_with_len_converter mb2wchar_with_len; /* convert a multibyte
+ * string to a wchar */
+ mblen_converter mblen; /* get byte length of a char */
+ mbdisplaylen_converter dsplen; /* get display width of a char */
+ mbverifier mbverify; /* verify multibyte sequence */
+ int maxmblen; /* max bytes for a char in this encoding */
+} pg_wchar_tbl;
+
+extern pg_wchar_tbl pg_wchar_table[];
+
+/*
+ * UTF-8 to local code conversion map
+ * Note that we limit the max length of UTF-8 to 4 bytes,
+ * which is UCS-4 00010000-001FFFFF range.
+ */
+typedef struct
+{
+ uint32 utf; /* UTF-8 */
+ uint32 code; /* local code */
+} pg_utf_to_local;
+
+/*
+ * local code to UTF-8 conversion map
+ */
+typedef struct
+{
+ uint32 code; /* local code */
+ uint32 utf; /* UTF-8 */
+} pg_local_to_utf;
+
+/*
+ * UTF-8 to local code conversion map(combined characters)
+ */
+typedef struct
+{
+ uint32 utf1; /* UTF-8 code 1 */
+ uint32 utf2; /* UTF-8 code 2 */
+ uint32 code; /* local code */
+} pg_utf_to_local_combined;
+
+/*
+ * local code to UTF-8 conversion map(combined characters)
+ */
+typedef struct
+{
+ uint32 code; /* local code */
+ uint32 utf1; /* UTF-8 code 1 */
+ uint32 utf2; /* UTF-8 code 2 */
+} pg_local_to_utf_combined;
+
+/*
+ * Support macro for encoding conversion functions to validate their
+ * arguments. (This could be made more compact if we included fmgr.h
+ * here, but we don't want to do that because this header file is also
+ * used by frontends.)
+ */
+#define CHECK_ENCODING_CONVERSION_ARGS(srcencoding,destencoding) \
+ check_encoding_conversion_args(PG_GETARG_INT32(0), \
+ PG_GETARG_INT32(1), \
+ PG_GETARG_INT32(4), \
+ (srcencoding), \
+ (destencoding))
+
+
+/*
+ * These functions are considered part of libpq's exported API and
+ * are also declared in libpq-fe.h.
+ */
+extern int pg_char_to_encoding(const char *name);
+extern const char *pg_encoding_to_char(int encoding);
+extern int pg_valid_server_encoding_id(int encoding);
+
+/*
+ * Remaining functions are not considered part of libpq's API, though many
+ * of them do exist inside libpq.
+ */
+extern pg_encname *pg_char_to_encname_struct(const char *name);
+
+extern int pg_mb2wchar(const char *from, pg_wchar *to);
+extern int pg_mb2wchar_with_len(const char *from, pg_wchar *to, int len);
+extern int pg_encoding_mb2wchar_with_len(int encoding,
+ const char *from, pg_wchar *to, int len);
+extern int pg_char_and_wchar_strcmp(const char *s1, const pg_wchar *s2);
+extern int pg_wchar_strncmp(const pg_wchar *s1, const pg_wchar *s2, size_t n);
+extern int pg_char_and_wchar_strncmp(const char *s1, const pg_wchar *s2, size_t n);
+extern size_t pg_wchar_strlen(const pg_wchar *wstr);
+extern int pg_mblen(const char *mbstr);
+extern int pg_dsplen(const char *mbstr);
+extern int pg_encoding_mblen(int encoding, const char *mbstr);
+extern int pg_encoding_dsplen(int encoding, const char *mbstr);
+extern int pg_encoding_verifymb(int encoding, const char *mbstr, int len);
+extern int pg_mule_mblen(const unsigned char *mbstr);
+extern int pg_mic_mblen(const unsigned char *mbstr);
+extern int pg_mbstrlen(const char *mbstr);
+extern int pg_mbstrlen_with_len(const char *mbstr, int len);
+extern int pg_mbcliplen(const char *mbstr, int len, int limit);
+extern int pg_encoding_mbcliplen(int encoding, const char *mbstr,
+ int len, int limit);
+extern int pg_mbcharcliplen(const char *mbstr, int len, int imit);
+extern int pg_encoding_max_length(int encoding);
+extern int pg_database_encoding_max_length(void);
+
+#ifdef USE_WIDE_UPPER_LOWER
+extern size_t wchar2char(char *to, const wchar_t *from, size_t tolen);
+extern size_t char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen);
+#endif
+
+extern int SetClientEncoding(int encoding, bool doit);
+extern void InitializeClientEncoding(void);
+extern int pg_get_client_encoding(void);
+extern const char *pg_get_client_encoding_name(void);
+
+extern void SetDatabaseEncoding(int encoding);
+extern int GetDatabaseEncoding(void);
+extern const char *GetDatabaseEncodingName(void);
+extern int GetPlatformEncoding(void);
+extern void pg_bind_textdomain_codeset(const char *domainname);
+
+extern int pg_valid_client_encoding(const char *name);
+extern int pg_valid_server_encoding(const char *name);
+
+extern unsigned char *unicode_to_utf8(pg_wchar c, unsigned char *utf8string);
+extern int pg_utf_mblen(const unsigned char *);
+extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len,
+ int src_encoding,
+ int dest_encoding);
+
+extern char *pg_client_to_server(const char *s, int len);
+extern char *pg_server_to_client(const char *s, int len);
+
+extern unsigned short BIG5toCNS(unsigned short big5, unsigned char *lc);
+extern unsigned short CNStoBIG5(unsigned short cns, unsigned char lc);
+
+extern void LocalToUtf(const unsigned char *iso, unsigned char *utf,
+ const pg_local_to_utf *map, const pg_local_to_utf_combined *cmap,
+ int size1, int size2, int encoding, int len);
+
+extern void UtfToLocal(const unsigned char *utf, unsigned char *iso,
+ const pg_utf_to_local *map, const pg_utf_to_local_combined *cmap,
+ int size1, int size2, int encoding, int len);
+
+extern bool pg_verifymbstr(const char *mbstr, int len, bool noError);
+extern bool pg_verify_mbstr(int encoding, const char *mbstr, int len,
+ bool noError);
+extern int pg_verify_mbstr_len(int encoding, const char *mbstr, int len,
+ bool noError);
+
+extern void check_encoding_conversion_args(int src_encoding,
+ int dest_encoding,
+ int len,
+ int expected_src_encoding,
+ int expected_dest_encoding);
+
+extern void report_invalid_encoding(int encoding, const char *mbstr, int len);
+extern void report_untranslatable_char(int src_encoding, int dest_encoding,
+ const char *mbstr, int len);
+
+extern void pg_ascii2mic(const unsigned char *l, unsigned char *p, int len);
+extern void pg_mic2ascii(const unsigned char *mic, unsigned char *p, int len);
+extern void latin2mic(const unsigned char *l, unsigned char *p, int len,
+ int lc, int encoding);
+extern void mic2latin(const unsigned char *mic, unsigned char *p, int len,
+ int lc, int encoding);
+extern void latin2mic_with_table(const unsigned char *l, unsigned char *p,
+ int len, int lc, int encoding,
+ const unsigned char *tab);
+extern void mic2latin_with_table(const unsigned char *mic, unsigned char *p,
+ int len, int lc, int encoding,
+ const unsigned char *tab);
+
+extern bool pg_utf8_islegal(const unsigned char *source, int length);
+
+#ifdef WIN32
+extern WCHAR *pgwin32_toUTF16(const char *str, int len, int *utf16len);
+#endif
+
+#endif /* PG_WCHAR_H */
diff --git a/parser/scanner.h b/parser/scanner.h
new file mode 100644
index 000000000..ed3b42b55
--- /dev/null
+++ b/parser/scanner.h
@@ -0,0 +1,119 @@
+/*-------------------------------------------------------------------------
+ *
+ * scanner.h
+ * API for the core scanner (flex machine)
+ *
+ * The core scanner is also used by PL/pgsql, so we provide a public API
+ * for it. However, the rest of the backend is only expected to use the
+ * higher-level API provided by parser.h.
+ *
+ *
+ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * $PostgreSQL: pgsql/src/include/parser/scanner.h,v 1.3 2010/02/26 02:01:27 momjian Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef SCANNER_H
+#define SCANNER_H
+
+#include "keywords.h"
+
+/*
+ * The scanner returns extra data about scanned tokens in this union type.
+ * Note that this is a subset of the fields used in YYSTYPE of the bison
+ * parsers built atop the scanner.
+ */
+typedef union core_YYSTYPE
+{
+ int ival; /* for integer literals */
+ char *str; /* for identifiers and non-integer literals */
+ const char *keyword; /* canonical spelling of keywords */
+} core_YYSTYPE;
+
+/*
+ * We track token locations in terms of byte offsets from the start of the
+ * source string, not the column number/line number representation that
+ * bison uses by default. Also, to minimize overhead we track only one
+ * location (usually the first token location) for each construct, not
+ * the beginning and ending locations as bison does by default. It's
+ * therefore sufficient to make YYLTYPE an int.
+ */
+#define YYLTYPE int
+
+/*
+ * Another important component of the scanner's API is the token code numbers.
+ * However, those are not defined in this file, because bison insists on
+ * defining them for itself. The token codes used by the core scanner are
+ * the ASCII characters plus these:
+ * %token <str> IDENT FCONST SCONST BCONST XCONST Op
+ * %token <ival> ICONST PARAM
+ * %token TYPECAST DOT_DOT COLON_EQUALS
+ * The above token definitions *must* be the first ones declared in any
+ * bison parser built atop this scanner, so that they will have consistent
+ * numbers assigned to them (specifically, IDENT = 258 and so on).
+ */
+
+/*
+ * The YY_EXTRA data that a flex scanner allows us to pass around.
+ * Private state needed by the core scanner goes here. Note that the actual
+ * yy_extra struct may be larger and have this as its first component, thus
+ * allowing the calling parser to keep some fields of its own in YY_EXTRA.
+ */
+typedef struct core_yy_extra_type
+{
+ /*
+ * The string the scanner is physically scanning. We keep this mainly so
+ * that we can cheaply compute the offset of the current token (yytext).
+ */
+ char *scanbuf;
+ Size scanbuflen;
+
+ /*
+ * The keyword list to use.
+ */
+ const ScanKeyword *keywords;
+ int num_keywords;
+
+ /*
+ * literalbuf is used to accumulate literal values when multiple rules are
+ * needed to parse a single literal. Call startlit() to reset buffer to
+ * empty, addlit() to add text. NOTE: the string in literalbuf is NOT
+ * necessarily null-terminated, but there always IS room to add a trailing
+ * null at offset literallen. We store a null only when we need it.
+ */
+ char *literalbuf; /* palloc'd expandable buffer */
+ int literallen; /* actual current string length */
+ int literalalloc; /* current allocated buffer size */
+
+ int xcdepth; /* depth of nesting in slash-star comments */
+ char *dolqstart; /* current $foo$ quote start string */
+
+ /* first part of UTF16 surrogate pair for Unicode escapes */
+ int32 utf16_first_part;
+
+ /* state variables for literal-lexing warnings */
+ bool warn_on_first_escape;
+ bool saw_non_ascii;
+} core_yy_extra_type;
+
+/*
+ * The type of yyscanner is opaque outside scan.l.
+ */
+typedef void *core_yyscan_t;
+
+
+/* Entry points in parser/scan.l */
+extern core_yyscan_t scanner_init(const char *str,
+ core_yy_extra_type *yyext,
+ const ScanKeyword *keywords,
+ int num_keywords);
+extern void scanner_finish(core_yyscan_t yyscanner);
+extern int core_yylex(core_YYSTYPE *lvalp, YYLTYPE *llocp,
+ core_yyscan_t yyscanner);
+extern int scanner_errposition(int location, core_yyscan_t yyscanner);
+extern void scanner_yyerror(const char *message, core_yyscan_t yyscanner);
+
+#endif /* SCANNER_H */
diff --git a/parser/scansup.c b/parser/scansup.c
new file mode 100644
index 000000000..2cc018b04
--- /dev/null
+++ b/parser/scansup.c
@@ -0,0 +1,220 @@
+/*-------------------------------------------------------------------------
+ *
+ * scansup.c
+ * support routines for the lex/flex scanner, used by both the normal
+ * backend as well as the bootstrap backend
+ *
+ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * $PostgreSQL: pgsql/src/backend/parser/scansup.c,v 1.42 2010/07/06 19:18:57 momjian Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "pool_parser.h"
+#include "pool_memory.h"
+
+#include <ctype.h>
+#include <string.h>
+
+#include "scansup.h"
+#include "pg_wchar.h"
+
+
+/* ----------------
+ * scanstr
+ *
+ * if the string passed in has escaped codes, map the escape codes to actual
+ * chars
+ *
+ * the string returned is palloc'd and should eventually be pfree'd by the
+ * caller!
+ * ----------------
+ */
+char *
+scanstr(const char *s)
+{
+ char *newStr;
+ int len,
+ i,
+ j;
+
+ if (s == NULL || s[0] == '\0')
+ return pstrdup("");
+
+ len = strlen(s);
+
+ newStr = palloc(len + 1); /* string cannot get longer */
+
+ for (i = 0, j = 0; i < len; i++)
+ {
+ if (s[i] == '\'')
+ {
+ /*
+ * Note: if scanner is working right, unescaped quotes can only
+ * appear in pairs, so there should be another character.
+ */
+ i++;
+ newStr[j] = s[i];
+ }
+ else if (s[i] == '\\')
+ {
+ i++;
+ switch (s[i])
+ {
+ case 'b':
+ newStr[j] = '\b';
+ break;
+ case 'f':
+ newStr[j] = '\f';
+ break;
+ case 'n':
+ newStr[j] = '\n';
+ break;
+ case 'r':
+ newStr[j] = '\r';
+ break;
+ case 't':
+ newStr[j] = '\t';
+ break;
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ {
+ int k;
+ long octVal = 0;
+
+ for (k = 0;
+ s[i + k] >= '0' && s[i + k] <= '7' && k < 3;
+ k++)
+ octVal = (octVal << 3) + (s[i + k] - '0');
+ i += k - 1;
+ newStr[j] = ((char) octVal);
+ }
+ break;
+ default:
+ newStr[j] = s[i];
+ break;
+ } /* switch */
+ } /* s[i] == '\\' */
+ else
+ newStr[j] = s[i];
+ j++;
+ }
+ newStr[j] = '\0';
+ return newStr;
+}
+
+
+/*
+ * downcase_truncate_identifier() --- do appropriate downcasing and
+ * truncation of an unquoted identifier. Optionally warn of truncation.
+ *
+ * Returns a palloc'd string containing the adjusted identifier.
+ *
+ * Note: in some usages the passed string is not null-terminated.
+ *
+ * Note: the API of this function is designed to allow for downcasing
+ * transformations that increase the string length, but we don't yet
+ * support that. If you want to implement it, you'll need to fix
+ * SplitIdentifierString() in utils/adt/varlena.c.
+ */
+char *
+downcase_truncate_identifier(const char *ident, int len, bool warn)
+{
+ char *result;
+ int i;
+
+ result = palloc(len + 1);
+
+ /*
+ * SQL99 specifies Unicode-aware case normalization, which we don't yet
+ * have the infrastructure for. Instead we use tolower() to provide a
+ * locale-aware translation. However, there are some locales where this
+ * is not right either (eg, Turkish may do strange things with 'i' and
+ * 'I'). Our current compromise is to use tolower() for characters with
+ * the high bit set, and use an ASCII-only downcasing for 7-bit
+ * characters.
+ */
+ for (i = 0; i < len; i++)
+ {
+ unsigned char ch = (unsigned char) ident[i];
+
+ if (ch >= 'A' && ch <= 'Z')
+ ch += 'a' - 'A';
+ else if (IS_HIGHBIT_SET(ch) && isupper(ch))
+ ch = tolower(ch);
+ result[i] = (char) ch;
+ }
+ result[i] = '\0';
+
+ if (i >= NAMEDATALEN)
+ truncate_identifier(result, i, warn);
+
+ return result;
+}
+
+/*
+ * truncate_identifier() --- truncate an identifier to NAMEDATALEN-1 bytes.
+ *
+ * The given string is modified in-place, if necessary. A warning is
+ * issued if requested.
+ *
+ * We require the caller to pass in the string length since this saves a
+ * strlen() call in some common usages.
+ */
+void
+truncate_identifier(char *ident, int len, bool warn)
+{
+#if PGPOOL_NOT_USED
+ if (len >= NAMEDATALEN)
+ {
+ len = pg_mbcliplen(ident, len, NAMEDATALEN - 1);
+ if (warn)
+ {
+ /*
+ * We avoid using %.*s here because it can misbehave if the data
+ * is not valid in what libc thinks is the prevailing encoding.
+ */
+ char buf[NAMEDATALEN];
+
+ memcpy(buf, ident, len);
+ buf[len] = '\0';
+ ereport(NOTICE,
+ (errcode(ERRCODE_NAME_TOO_LONG),
+ errmsg("identifier \"%s\" will be truncated to \"%s\"",
+ ident, buf)));
+ }
+ ident[len] = '\0';
+ }
+#endif
+}
+
+/*
+ * scanner_isspace() --- return TRUE if flex scanner considers char whitespace
+ *
+ * This should be used instead of the potentially locale-dependent isspace()
+ * function when it's important to match the lexer's behavior.
+ *
+ * In principle we might need similar functions for isalnum etc, but for the
+ * moment only isspace seems needed.
+ */
+bool
+scanner_isspace(char ch)
+{
+ /* This must match scan.l's list of {space} characters */
+ if (ch == ' ' ||
+ ch == '\t' ||
+ ch == '\n' ||
+ ch == '\r' ||
+ ch == '\f')
+ return true;
+ return false;
+}
diff --git a/parser/wchar.c b/parser/wchar.c
new file mode 100644
index 000000000..5f904cc68
--- /dev/null
+++ b/parser/wchar.c
@@ -0,0 +1,1640 @@
+/*
+ * conversion functions between pg_wchar and multibyte streams.
+ * Tatsuo Ishii
+ * $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.74 2010/01/04 20:38:31 adunstan Exp $
+ *
+ */
+/* can be used in either frontend or backend */
+#include "pool_parser.h"
+#include <stdio.h>
+#include <string.h>
+#include "pg_wchar.h"
+
+
+/*
+ * conversion to pg_wchar is done by "table driven."
+ * to add an encoding support, define mb2wchar_with_len(), mblen(), dsplen()
+ * for the particular encoding. Note that if the encoding is only
+ * supported in the client, you don't need to define
+ * mb2wchar_with_len() function (SJIS is the case).
+ *
+ * These functions generally assume that their input is validly formed.
+ * The "verifier" functions, further down in the file, have to be more
+ * paranoid. We expect that mblen() does not need to examine more than
+ * the first byte of the character to discover the correct length.
+ *
+ * Note: for the display output of psql to work properly, the return values
+ * of the dsplen functions must conform to the Unicode standard. In particular
+ * the NUL character is zero width and control characters are generally
+ * width -1. It is recommended that non-ASCII encodings refer their ASCII
+ * subset to the ASCII routines to ensure consistency.
+ */
+
+/*
+ * SQL/ASCII
+ */
+static int
+pg_ascii2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
+{
+ int cnt = 0;
+
+ while (len > 0 && *from)
+ {
+ *to++ = *from++;
+ len--;
+ cnt++;
+ }
+ *to = 0;
+ return cnt;
+}
+
+static int
+pg_ascii_mblen(const unsigned char *s)
+{
+ return 1;
+}
+
+static int
+pg_ascii_dsplen(const unsigned char *s)
+{
+ if (*s == '\0')
+ return 0;
+ if (*s < 0x20 || *s == 0x7f)
+ return -1;
+
+ return 1;
+}
+
+/*
+ * EUC
+ */
+static int
+pg_euc2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
+{
+ int cnt = 0;
+
+ while (len > 0 && *from)
+ {
+ if (*from == SS2 && len >= 2) /* JIS X 0201 (so called "1 byte
+ * KANA") */
+ {
+ from++;
+ *to = (SS2 << 8) | *from++;
+ len -= 2;
+ }
+ else if (*from == SS3 && len >= 3) /* JIS X 0212 KANJI */
+ {
+ from++;
+ *to = (SS3 << 16) | (*from++ << 8);
+ *to |= *from++;
+ len -= 3;
+ }
+ else if (IS_HIGHBIT_SET(*from) && len >= 2) /* JIS X 0208 KANJI */
+ {
+ *to = *from++ << 8;
+ *to |= *from++;
+ len -= 2;
+ }
+ else
+ /* must be ASCII */
+ {
+ *to = *from++;
+ len--;
+ }
+ to++;
+ cnt++;
+ }
+ *to = 0;
+ return cnt;
+}
+
+static inline int
+pg_euc_mblen(const unsigned char *s)
+{
+ int len;
+
+ if (*s == SS2)
+ len = 2;
+ else if (*s == SS3)
+ len = 3;
+ else if (IS_HIGHBIT_SET(*s))
+ len = 2;
+ else
+ len = 1;
+ return len;
+}
+
+static inline int
+pg_euc_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (*s == SS2)
+ len = 2;
+ else if (*s == SS3)
+ len = 2;
+ else if (IS_HIGHBIT_SET(*s))
+ len = 2;
+ else
+ len = pg_ascii_dsplen(s);
+ return len;
+}
+
+/*
+ * EUC_JP
+ */
+static int
+pg_eucjp2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
+{
+ return pg_euc2wchar_with_len(from, to, len);
+}
+
+static int
+pg_eucjp_mblen(const unsigned char *s)
+{
+ return pg_euc_mblen(s);
+}
+
+static int
+pg_eucjp_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (*s == SS2)
+ len = 1;
+ else if (*s == SS3)
+ len = 2;
+ else if (IS_HIGHBIT_SET(*s))
+ len = 2;
+ else
+ len = pg_ascii_dsplen(s);
+ return len;
+}
+
+/*
+ * EUC_KR
+ */
+static int
+pg_euckr2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
+{
+ return pg_euc2wchar_with_len(from, to, len);
+}
+
+static int
+pg_euckr_mblen(const unsigned char *s)
+{
+ return pg_euc_mblen(s);
+}
+
+static int
+pg_euckr_dsplen(const unsigned char *s)
+{
+ return pg_euc_dsplen(s);
+}
+
+/*
+ * EUC_CN
+ *
+ */
+static int
+pg_euccn2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
+{
+ int cnt = 0;
+
+ while (len > 0 && *from)
+ {
+ if (*from == SS2 && len >= 3) /* code set 2 (unused?) */
+ {
+ from++;
+ *to = (SS2 << 16) | (*from++ << 8);
+ *to |= *from++;
+ len -= 3;
+ }
+ else if (*from == SS3 && len >= 3) /* code set 3 (unsed ?) */
+ {
+ from++;
+ *to = (SS3 << 16) | (*from++ << 8);
+ *to |= *from++;
+ len -= 3;
+ }
+ else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 1 */
+ {
+ *to = *from++ << 8;
+ *to |= *from++;
+ len -= 2;
+ }
+ else
+ {
+ *to = *from++;
+ len--;
+ }
+ to++;
+ cnt++;
+ }
+ *to = 0;
+ return cnt;
+}
+
+static int
+pg_euccn_mblen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_HIGHBIT_SET(*s))
+ len = 2;
+ else
+ len = 1;
+ return len;
+}
+
+static int
+pg_euccn_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_HIGHBIT_SET(*s))
+ len = 2;
+ else
+ len = pg_ascii_dsplen(s);
+ return len;
+}
+
+/*
+ * EUC_TW
+ *
+ */
+static int
+pg_euctw2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
+{
+ int cnt = 0;
+
+ while (len > 0 && *from)
+ {
+ if (*from == SS2 && len >= 4) /* code set 2 */
+ {
+ from++;
+ *to = (((uint32) SS2) << 24) | (*from++ << 16);
+ *to |= *from++ << 8;
+ *to |= *from++;
+ len -= 4;
+ }
+ else if (*from == SS3 && len >= 3) /* code set 3 (unused?) */
+ {
+ from++;
+ *to = (SS3 << 16) | (*from++ << 8);
+ *to |= *from++;
+ len -= 3;
+ }
+ else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 2 */
+ {
+ *to = *from++ << 8;
+ *to |= *from++;
+ len -= 2;
+ }
+ else
+ {
+ *to = *from++;
+ len--;
+ }
+ to++;
+ cnt++;
+ }
+ *to = 0;
+ return cnt;
+}
+
+static int
+pg_euctw_mblen(const unsigned char *s)
+{
+ int len;
+
+ if (*s == SS2)
+ len = 4;
+ else if (*s == SS3)
+ len = 3;
+ else if (IS_HIGHBIT_SET(*s))
+ len = 2;
+ else
+ len = 1;
+ return len;
+}
+
+static int
+pg_euctw_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (*s == SS2)
+ len = 2;
+ else if (*s == SS3)
+ len = 2;
+ else if (IS_HIGHBIT_SET(*s))
+ len = 2;
+ else
+ len = pg_ascii_dsplen(s);
+ return len;
+}
+
+/*
+ * JOHAB
+ */
+static int
+pg_johab_mblen(const unsigned char *s)
+{
+ return pg_euc_mblen(s);
+}
+
+static int
+pg_johab_dsplen(const unsigned char *s)
+{
+ return pg_euc_dsplen(s);
+}
+
+/*
+ * convert UTF8 string to pg_wchar (UCS-4)
+ * caller must allocate enough space for "to", including a trailing zero!
+ * len: length of from.
+ * "from" not necessarily null terminated.
+ */
+static int
+pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
+{
+ int cnt = 0;
+ uint32 c1,
+ c2,
+ c3,
+ c4;
+
+ while (len > 0 && *from)
+ {
+ if ((*from & 0x80) == 0)
+ {
+ *to = *from++;
+ len--;
+ }
+ else if ((*from & 0xe0) == 0xc0)
+ {
+ if (len < 2)
+ break; /* drop trailing incomplete char */
+ c1 = *from++ & 0x1f;
+ c2 = *from++ & 0x3f;
+ *to = (c1 << 6) | c2;
+ len -= 2;
+ }
+ else if ((*from & 0xf0) == 0xe0)
+ {
+ if (len < 3)
+ break; /* drop trailing incomplete char */
+ c1 = *from++ & 0x0f;
+ c2 = *from++ & 0x3f;
+ c3 = *from++ & 0x3f;
+ *to = (c1 << 12) | (c2 << 6) | c3;
+ len -= 3;
+ }
+ else if ((*from & 0xf8) == 0xf0)
+ {
+ if (len < 4)
+ break; /* drop trailing incomplete char */
+ c1 = *from++ & 0x07;
+ c2 = *from++ & 0x3f;
+ c3 = *from++ & 0x3f;
+ c4 = *from++ & 0x3f;
+ *to = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
+ len -= 4;
+ }
+ else
+ {
+ /* treat a bogus char as length 1; not ours to raise error */
+ *to = *from++;
+ len--;
+ }
+ to++;
+ cnt++;
+ }
+ *to = 0;
+ return cnt;
+}
+
+
+/*
+ * Map a Unicode code point to UTF-8. utf8string must have 4 bytes of
+ * space allocated.
+ */
+unsigned char *
+unicode_to_utf8(pg_wchar c, unsigned char *utf8string)
+{
+ if (c <= 0x7F)
+ {
+ utf8string[0] = c;
+ }
+ else if (c <= 0x7FF)
+ {
+ utf8string[0] = 0xC0 | ((c >> 6) & 0x1F);
+ utf8string[1] = 0x80 | (c & 0x3F);
+ }
+ else if (c <= 0xFFFF)
+ {
+ utf8string[0] = 0xE0 | ((c >> 12) & 0x0F);
+ utf8string[1] = 0x80 | ((c >> 6) & 0x3F);
+ utf8string[2] = 0x80 | (c & 0x3F);
+ }
+ else
+ {
+ utf8string[0] = 0xF0 | ((c >> 18) & 0x07);
+ utf8string[1] = 0x80 | ((c >> 12) & 0x3F);
+ utf8string[2] = 0x80 | ((c >> 6) & 0x3F);
+ utf8string[3] = 0x80 | (c & 0x3F);
+ }
+
+ return utf8string;
+}
+
+
+/*
+ * Return the byte length of a UTF8 character pointed to by s
+ *
+ * Note: in the current implementation we do not support UTF8 sequences
+ * of more than 4 bytes; hence do NOT return a value larger than 4.
+ * We return "1" for any leading byte that is either flat-out illegal or
+ * indicates a length larger than we support.
+ *
+ * pg_utf2wchar_with_len(), utf2ucs(), pg_utf8_islegal(), and perhaps
+ * other places would need to be fixed to change this.
+ */
+int
+pg_utf_mblen(const unsigned char *s)
+{
+ int len;
+
+ if ((*s & 0x80) == 0)
+ len = 1;
+ else if ((*s & 0xe0) == 0xc0)
+ len = 2;
+ else if ((*s & 0xf0) == 0xe0)
+ len = 3;
+ else if ((*s & 0xf8) == 0xf0)
+ len = 4;
+#ifdef NOT_USED
+ else if ((*s & 0xfc) == 0xf8)
+ len = 5;
+ else if ((*s & 0xfe) == 0xfc)
+ len = 6;
+#endif
+ else
+ len = 1;
+ return len;
+}
+
+/*
+ * This is an implementation of wcwidth() and wcswidth() as defined in
+ * "The Single UNIX Specification, Version 2, The Open Group, 1997"
+ * <http://www.UNIX-systems.org/online.html>
+ *
+ * Markus Kuhn -- 2001-09-08 -- public domain
+ *
+ * customised for PostgreSQL
+ *
+ * original available at : http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+ */
+
+struct mbinterval
+{
+ unsigned short first;
+ unsigned short last;
+};
+
+/* auxiliary function for binary search in interval table */
+static int
+mbbisearch(pg_wchar ucs, const struct mbinterval * table, int max)
+{
+ int min = 0;
+ int mid;
+
+ if (ucs < table[0].first || ucs > table[max].last)
+ return 0;
+ while (max >= min)
+ {
+ mid = (min + max) / 2;
+ if (ucs > table[mid].last)
+ min = mid + 1;
+ else if (ucs < table[mid].first)
+ max = mid - 1;
+ else
+ return 1;
+ }
+
+ return 0;
+}
+
+
+/* The following functions define the column width of an ISO 10646
+ * character as follows:
+ *
+ * - The null character (U+0000) has a column width of 0.
+ *
+ * - Other C0/C1 control characters and DEL will lead to a return
+ * value of -1.
+ *
+ * - Non-spacing and enclosing combining characters (general
+ * category code Mn or Me in the Unicode database) have a
+ * column width of 0.
+ *
+ * - Other format characters (general category code Cf in the Unicode
+ * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
+ *
+ * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
+ * have a column width of 0.
+ *
+ * - Spacing characters in the East Asian Wide (W) or East Asian
+ * FullWidth (F) category as defined in Unicode Technical
+ * Report #11 have a column width of 2.
+ *
+ * - All remaining characters (including all printable
+ * ISO 8859-1 and WGL4 characters, Unicode control characters,
+ * etc.) have a column width of 1.
+ *
+ * This implementation assumes that wchar_t characters are encoded
+ * in ISO 10646.
+ */
+
+static int
+ucs_wcwidth(pg_wchar ucs)
+{
+ /* sorted list of non-overlapping intervals of non-spacing characters */
+ static const struct mbinterval combining[] = {
+ {0x0300, 0x034E}, {0x0360, 0x0362}, {0x0483, 0x0486},
+ {0x0488, 0x0489}, {0x0591, 0x05A1}, {0x05A3, 0x05B9},
+ {0x05BB, 0x05BD}, {0x05BF, 0x05BF}, {0x05C1, 0x05C2},
+ {0x05C4, 0x05C4}, {0x064B, 0x0655}, {0x0670, 0x0670},
+ {0x06D6, 0x06E4}, {0x06E7, 0x06E8}, {0x06EA, 0x06ED},
+ {0x070F, 0x070F}, {0x0711, 0x0711}, {0x0730, 0x074A},
+ {0x07A6, 0x07B0}, {0x0901, 0x0902}, {0x093C, 0x093C},
+ {0x0941, 0x0948}, {0x094D, 0x094D}, {0x0951, 0x0954},
+ {0x0962, 0x0963}, {0x0981, 0x0981}, {0x09BC, 0x09BC},
+ {0x09C1, 0x09C4}, {0x09CD, 0x09CD}, {0x09E2, 0x09E3},
+ {0x0A02, 0x0A02}, {0x0A3C, 0x0A3C}, {0x0A41, 0x0A42},
+ {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A70, 0x0A71},
+ {0x0A81, 0x0A82}, {0x0ABC, 0x0ABC}, {0x0AC1, 0x0AC5},
+ {0x0AC7, 0x0AC8}, {0x0ACD, 0x0ACD}, {0x0B01, 0x0B01},
+ {0x0B3C, 0x0B3C}, {0x0B3F, 0x0B3F}, {0x0B41, 0x0B43},
+ {0x0B4D, 0x0B4D}, {0x0B56, 0x0B56}, {0x0B82, 0x0B82},
+ {0x0BC0, 0x0BC0}, {0x0BCD, 0x0BCD}, {0x0C3E, 0x0C40},
+ {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56},
+ {0x0CBF, 0x0CBF}, {0x0CC6, 0x0CC6}, {0x0CCC, 0x0CCD},
+ {0x0D41, 0x0D43}, {0x0D4D, 0x0D4D}, {0x0DCA, 0x0DCA},
+ {0x0DD2, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0E31, 0x0E31},
+ {0x0E34, 0x0E3A}, {0x0E47, 0x0E4E}, {0x0EB1, 0x0EB1},
+ {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC}, {0x0EC8, 0x0ECD},
+ {0x0F18, 0x0F19}, {0x0F35, 0x0F35}, {0x0F37, 0x0F37},
+ {0x0F39, 0x0F39}, {0x0F71, 0x0F7E}, {0x0F80, 0x0F84},
+ {0x0F86, 0x0F87}, {0x0F90, 0x0F97}, {0x0F99, 0x0FBC},
+ {0x0FC6, 0x0FC6}, {0x102D, 0x1030}, {0x1032, 0x1032},
+ {0x1036, 0x1037}, {0x1039, 0x1039}, {0x1058, 0x1059},
+ {0x1160, 0x11FF}, {0x17B7, 0x17BD}, {0x17C6, 0x17C6},
+ {0x17C9, 0x17D3}, {0x180B, 0x180E}, {0x18A9, 0x18A9},
+ {0x200B, 0x200F}, {0x202A, 0x202E}, {0x206A, 0x206F},
+ {0x20D0, 0x20E3}, {0x302A, 0x302F}, {0x3099, 0x309A},
+ {0xFB1E, 0xFB1E}, {0xFE20, 0xFE23}, {0xFEFF, 0xFEFF},
+ {0xFFF9, 0xFFFB}
+ };
+
+ /* test for 8-bit control characters */
+ if (ucs == 0)
+ return 0;
+
+ if (ucs < 0x20 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff)
+ return -1;
+
+ /* binary search in table of non-spacing characters */
+ if (mbbisearch(ucs, combining,
+ sizeof(combining) / sizeof(struct mbinterval) - 1))
+ return 0;
+
+ /*
+ * if we arrive here, ucs is not a combining or C0/C1 control character
+ */
+
+ return 1 +
+ (ucs >= 0x1100 &&
+ (ucs <= 0x115f || /* Hangul Jamo init. consonants */
+ (ucs >= 0x2e80 && ucs <= 0xa4cf && (ucs & ~0x0011) != 0x300a &&
+ ucs != 0x303f) || /* CJK ... Yi */
+ (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
+ (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility
+ * Ideographs */
+ (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
+ (ucs >= 0xff00 && ucs <= 0xff5f) || /* Fullwidth Forms */
+ (ucs >= 0xffe0 && ucs <= 0xffe6) ||
+ (ucs >= 0x20000 && ucs <= 0x2ffff)));
+}
+
+static pg_wchar
+utf2ucs(const unsigned char *c)
+{
+ /*
+ * one char version of pg_utf2wchar_with_len. no control here, c must
+ * point to a large enough string
+ */
+ if ((*c & 0x80) == 0)
+ return (pg_wchar) c[0];
+ else if ((*c & 0xe0) == 0xc0)
+ return (pg_wchar) (((c[0] & 0x1f) << 6) |
+ (c[1] & 0x3f));
+ else if ((*c & 0xf0) == 0xe0)
+ return (pg_wchar) (((c[0] & 0x0f) << 12) |
+ ((c[1] & 0x3f) << 6) |
+ (c[2] & 0x3f));
+ else if ((*c & 0xf8) == 0xf0)
+ return (pg_wchar) (((c[0] & 0x07) << 18) |
+ ((c[1] & 0x3f) << 12) |
+ ((c[2] & 0x3f) << 6) |
+ (c[3] & 0x3f));
+ else
+ /* that is an invalid code on purpose */
+ return 0xffffffff;
+}
+
+static int
+pg_utf_dsplen(const unsigned char *s)
+{
+ return ucs_wcwidth(utf2ucs(s));
+}
+
+/*
+ * convert mule internal code to pg_wchar
+ * caller should allocate enough space for "to"
+ * len: length of from.
+ * "from" not necessarily null terminated.
+ */
+static int
+pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
+{
+ int cnt = 0;
+
+ while (len > 0 && *from)
+ {
+ if (IS_LC1(*from) && len >= 2)
+ {
+ *to = *from++ << 16;
+ *to |= *from++;
+ len -= 2;
+ }
+ else if (IS_LCPRV1(*from) && len >= 3)
+ {
+ from++;
+ *to = *from++ << 16;
+ *to |= *from++;
+ len -= 3;
+ }
+ else if (IS_LC2(*from) && len >= 3)
+ {
+ *to = *from++ << 16;
+ *to |= *from++ << 8;
+ *to |= *from++;
+ len -= 3;
+ }
+ else if (IS_LCPRV2(*from) && len >= 4)
+ {
+ from++;
+ *to = *from++ << 16;
+ *to |= *from++ << 8;
+ *to |= *from++;
+ len -= 4;
+ }
+ else
+ { /* assume ASCII */
+ *to = (unsigned char) *from++;
+ len--;
+ }
+ to++;
+ cnt++;
+ }
+ *to = 0;
+ return cnt;
+}
+
+int
+pg_mule_mblen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_LC1(*s))
+ len = 2;
+ else if (IS_LCPRV1(*s))
+ len = 3;
+ else if (IS_LC2(*s))
+ len = 3;
+ else if (IS_LCPRV2(*s))
+ len = 4;
+ else
+ len = 1; /* assume ASCII */
+ return len;
+}
+
+static int
+pg_mule_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_LC1(*s))
+ len = 1;
+ else if (IS_LCPRV1(*s))
+ len = 1;
+ else if (IS_LC2(*s))
+ len = 2;
+ else if (IS_LCPRV2(*s))
+ len = 2;
+ else
+ len = 1; /* assume ASCII */
+
+ return len;
+}
+
+/*
+ * ISO8859-1
+ */
+static int
+pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
+{
+ int cnt = 0;
+
+ while (len > 0 && *from)
+ {
+ *to++ = *from++;
+ len--;
+ cnt++;
+ }
+ *to = 0;
+ return cnt;
+}
+
+static int
+pg_latin1_mblen(const unsigned char *s)
+{
+ return 1;
+}
+
+static int
+pg_latin1_dsplen(const unsigned char *s)
+{
+ return pg_ascii_dsplen(s);
+}
+
+/*
+ * SJIS
+ */
+static int
+pg_sjis_mblen(const unsigned char *s)
+{
+ int len;
+
+ if (*s >= 0xa1 && *s <= 0xdf)
+ len = 1; /* 1 byte kana? */
+ else if (IS_HIGHBIT_SET(*s))
+ len = 2; /* kanji? */
+ else
+ len = 1; /* should be ASCII */
+ return len;
+}
+
+static int
+pg_sjis_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (*s >= 0xa1 && *s <= 0xdf)
+ len = 1; /* 1 byte kana? */
+ else if (IS_HIGHBIT_SET(*s))
+ len = 2; /* kanji? */
+ else
+ len = pg_ascii_dsplen(s); /* should be ASCII */
+ return len;
+}
+
+/*
+ * Big5
+ */
+static int
+pg_big5_mblen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_HIGHBIT_SET(*s))
+ len = 2; /* kanji? */
+ else
+ len = 1; /* should be ASCII */
+ return len;
+}
+
+static int
+pg_big5_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_HIGHBIT_SET(*s))
+ len = 2; /* kanji? */
+ else
+ len = pg_ascii_dsplen(s); /* should be ASCII */
+ return len;
+}
+
+/*
+ * GBK
+ */
+static int
+pg_gbk_mblen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_HIGHBIT_SET(*s))
+ len = 2; /* kanji? */
+ else
+ len = 1; /* should be ASCII */
+ return len;
+}
+
+static int
+pg_gbk_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_HIGHBIT_SET(*s))
+ len = 2; /* kanji? */
+ else
+ len = pg_ascii_dsplen(s); /* should be ASCII */
+ return len;
+}
+
+/*
+ * UHC
+ */
+static int
+pg_uhc_mblen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_HIGHBIT_SET(*s))
+ len = 2; /* 2byte? */
+ else
+ len = 1; /* should be ASCII */
+ return len;
+}
+
+static int
+pg_uhc_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_HIGHBIT_SET(*s))
+ len = 2; /* 2byte? */
+ else
+ len = pg_ascii_dsplen(s); /* should be ASCII */
+ return len;
+}
+
+/*
+ * * GB18030
+ * * Added by Bill Huang <bhuang@redhat.com>,<bill_huanghb@ybb.ne.jp>
+ * */
+static int
+pg_gb18030_mblen(const unsigned char *s)
+{
+ int len;
+
+ if (!IS_HIGHBIT_SET(*s))
+ len = 1; /* ASCII */
+ else
+ {
+ if ((*(s + 1) >= 0x40 && *(s + 1) <= 0x7e) || (*(s + 1) >= 0x80 && *(s + 1) <= 0xfe))
+ len = 2;
+ else if (*(s + 1) >= 0x30 && *(s + 1) <= 0x39)
+ len = 4;
+ else
+ len = 2;
+ }
+ return len;
+}
+
+static int
+pg_gb18030_dsplen(const unsigned char *s)
+{
+ int len;
+
+ if (IS_HIGHBIT_SET(*s))
+ len = 2;
+ else
+ len = pg_ascii_dsplen(s); /* ASCII */
+ return len;
+}
+
+/*
+ *-------------------------------------------------------------------
+ * multibyte sequence validators
+ *
+ * These functions accept "s", a pointer to the first byte of a string,
+ * and "len", the remaining length of the string. If there is a validly
+ * encoded character beginning at *s, return its length in bytes; else
+ * return -1.
+ *
+ * The functions can assume that len > 0 and that *s != '\0', but they must
+ * test for and reject zeroes in any additional bytes of a multibyte character.
+ *
+ * Note that this definition allows the function for a single-byte
+ * encoding to be just "return 1".
+ *-------------------------------------------------------------------
+ */
+
+static int
+pg_ascii_verifier(const unsigned char *s, int len)
+{
+ return 1;
+}
+
+#define IS_EUC_RANGE_VALID(c) ((c) >= 0xa1 && (c) <= 0xfe)
+
+static int
+pg_eucjp_verifier(const unsigned char *s, int len)
+{
+ int l;
+ unsigned char c1,
+ c2;
+
+ c1 = *s++;
+
+ switch (c1)
+ {
+ case SS2: /* JIS X 0201 */
+ l = 2;
+ if (l > len)
+ return -1;
+ c2 = *s++;
+ if (c2 < 0xa1 || c2 > 0xdf)
+ return -1;
+ break;
+
+ case SS3: /* JIS X 0212 */
+ l = 3;
+ if (l > len)
+ return -1;
+ c2 = *s++;
+ if (!IS_EUC_RANGE_VALID(c2))
+ return -1;
+ c2 = *s++;
+ if (!IS_EUC_RANGE_VALID(c2))
+ return -1;
+ break;
+
+ default:
+ if (IS_HIGHBIT_SET(c1)) /* JIS X 0208? */
+ {
+ l = 2;
+ if (l > len)
+ return -1;
+ if (!IS_EUC_RANGE_VALID(c1))
+ return -1;
+ c2 = *s++;
+ if (!IS_EUC_RANGE_VALID(c2))
+ return -1;
+ }
+ else
+ /* must be ASCII */
+ {
+ l = 1;
+ }
+ break;
+ }
+
+ return l;
+}
+
+static int
+pg_euckr_verifier(const unsigned char *s, int len)
+{
+ int l;
+ unsigned char c1,
+ c2;
+
+ c1 = *s++;
+
+ if (IS_HIGHBIT_SET(c1))
+ {
+ l = 2;
+ if (l > len)
+ return -1;
+ if (!IS_EUC_RANGE_VALID(c1))
+ return -1;
+ c2 = *s++;
+ if (!IS_EUC_RANGE_VALID(c2))
+ return -1;
+ }
+ else
+ /* must be ASCII */
+ {
+ l = 1;
+ }
+
+ return l;
+}
+
+/* EUC-CN byte sequences are exactly same as EUC-KR */
+#define pg_euccn_verifier pg_euckr_verifier
+
+static int
+pg_euctw_verifier(const unsigned char *s, int len)
+{
+ int l;
+ unsigned char c1,
+ c2;
+
+ c1 = *s++;
+
+ switch (c1)
+ {
+ case SS2: /* CNS 11643 Plane 1-7 */
+ l = 4;
+ if (l > len)
+ return -1;
+ c2 = *s++;
+ if (c2 < 0xa1 || c2 > 0xa7)
+ return -1;
+ c2 = *s++;
+ if (!IS_EUC_RANGE_VALID(c2))
+ return -1;
+ c2 = *s++;
+ if (!IS_EUC_RANGE_VALID(c2))
+ return -1;
+ break;
+
+ case SS3: /* unused */
+ return -1;
+
+ default:
+ if (IS_HIGHBIT_SET(c1)) /* CNS 11643 Plane 1 */
+ {
+ l = 2;
+ if (l > len)
+ return -1;
+ /* no further range check on c1? */
+ c2 = *s++;
+ if (!IS_EUC_RANGE_VALID(c2))
+ return -1;
+ }
+ else
+ /* must be ASCII */
+ {
+ l = 1;
+ }
+ break;
+ }
+ return l;
+}
+
+static int
+pg_johab_verifier(const unsigned char *s, int len)
+{
+ int l,
+ mbl;
+ unsigned char c;
+
+ l = mbl = pg_johab_mblen(s);
+
+ if (len < l)
+ return -1;
+
+ if (!IS_HIGHBIT_SET(*s))
+ return mbl;
+
+ while (--l > 0)
+ {
+ c = *++s;
+ if (!IS_EUC_RANGE_VALID(c))
+ return -1;
+ }
+ return mbl;
+}
+
+static int
+pg_mule_verifier(const unsigned char *s, int len)
+{
+ int l,
+ mbl;
+ unsigned char c;
+
+ l = mbl = pg_mule_mblen(s);
+
+ if (len < l)
+ return -1;
+
+ while (--l > 0)
+ {
+ c = *++s;
+ if (!IS_HIGHBIT_SET(c))
+ return -1;
+ }
+ return mbl;
+}
+
+static int
+pg_latin1_verifier(const unsigned char *s, int len)
+{
+ return 1;
+}
+
+static int
+pg_sjis_verifier(const unsigned char *s, int len)
+{
+ int l,
+ mbl;
+ unsigned char c1,
+ c2;
+
+ l = mbl = pg_sjis_mblen(s);
+
+ if (len < l)
+ return -1;
+
+ if (l == 1) /* pg_sjis_mblen already verified it */
+ return mbl;
+
+ c1 = *s++;
+ c2 = *s;
+ if (!ISSJISHEAD(c1) || !ISSJISTAIL(c2))
+ return -1;
+ return mbl;
+}
+
+static int
+pg_big5_verifier(const unsigned char *s, int len)
+{
+ int l,
+ mbl;
+
+ l = mbl = pg_big5_mblen(s);
+
+ if (len < l)
+ return -1;
+
+ while (--l > 0)
+ {
+ if (*++s == '\0')
+ return -1;
+ }
+
+ return mbl;
+}
+
+static int
+pg_gbk_verifier(const unsigned char *s, int len)
+{
+ int l,
+ mbl;
+
+ l = mbl = pg_gbk_mblen(s);
+
+ if (len < l)
+ return -1;
+
+ while (--l > 0)
+ {
+ if (*++s == '\0')
+ return -1;
+ }
+
+ return mbl;
+}
+
+static int
+pg_uhc_verifier(const unsigned char *s, int len)
+{
+ int l,
+ mbl;
+
+ l = mbl = pg_uhc_mblen(s);
+
+ if (len < l)
+ return -1;
+
+ while (--l > 0)
+ {
+ if (*++s == '\0')
+ return -1;
+ }
+
+ return mbl;
+}
+
+static int
+pg_gb18030_verifier(const unsigned char *s, int len)
+{
+ int l,
+ mbl;
+
+ l = mbl = pg_gb18030_mblen(s);
+
+ if (len < l)
+ return -1;
+
+ while (--l > 0)
+ {
+ if (*++s == '\0')
+ return -1;
+ }
+
+ return mbl;
+}
+
+static int
+pg_utf8_verifier(const unsigned char *s, int len)
+{
+ int l = pg_utf_mblen(s);
+
+ if (len < l)
+ return -1;
+
+ if (!pg_utf8_islegal(s, l))
+ return -1;
+
+ return l;
+}
+
+/*
+ * Check for validity of a single UTF-8 encoded character
+ *
+ * This directly implements the rules in RFC3629. The bizarre-looking
+ * restrictions on the second byte are meant to ensure that there isn't
+ * more than one encoding of a given Unicode character point; that is,
+ * you may not use a longer-than-necessary byte sequence with high order
+ * zero bits to represent a character that would fit in fewer bytes.
+ * To do otherwise is to create security hazards (eg, create an apparent
+ * non-ASCII character that decodes to plain ASCII).
+ *
+ * length is assumed to have been obtained by pg_utf_mblen(), and the
+ * caller must have checked that that many bytes are present in the buffer.
+ */
+bool
+pg_utf8_islegal(const unsigned char *source, int length)
+{
+ unsigned char a;
+
+ switch (length)
+ {
+ default:
+ /* reject lengths 5 and 6 for now */
+ return false;
+ case 4:
+ a = source[3];
+ if (a < 0x80 || a > 0xBF)
+ return false;
+ /* FALL THRU */
+ case 3:
+ a = source[2];
+ if (a < 0x80 || a > 0xBF)
+ return false;
+ /* FALL THRU */
+ case 2:
+ a = source[1];
+ switch (*source)
+ {
+ case 0xE0:
+ if (a < 0xA0 || a > 0xBF)
+ return false;
+ break;
+ case 0xED:
+ if (a < 0x80 || a > 0x9F)
+ return false;
+ break;
+ case 0xF0:
+ if (a < 0x90 || a > 0xBF)
+ return false;
+ break;
+ case 0xF4:
+ if (a < 0x80 || a > 0x8F)
+ return false;
+ break;
+ default:
+ if (a < 0x80 || a > 0xBF)
+ return false;
+ break;
+ }
+ /* FALL THRU */
+ case 1:
+ a = *source;
+ if (a >= 0x80 && a < 0xC2)
+ return false;
+ if (a > 0xF4)
+ return false;
+ break;
+ }
+ return true;
+}
+
+/*
+ *-------------------------------------------------------------------
+ * encoding info table
+ * XXX must be sorted by the same order as enum pg_enc (in mb/pg_wchar.h)
+ *-------------------------------------------------------------------
+ */
+pg_wchar_tbl pg_wchar_table[] = {
+ {pg_ascii2wchar_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifier, 1}, /* PG_SQL_ASCII */
+ {pg_eucjp2wchar_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3}, /* PG_EUC_JP */
+ {pg_euccn2wchar_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifier, 2}, /* PG_EUC_CN */
+ {pg_euckr2wchar_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifier, 3}, /* PG_EUC_KR */
+ {pg_euctw2wchar_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifier, 4}, /* PG_EUC_TW */
+ {pg_eucjp2wchar_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3}, /* PG_EUC_JIS_2004 */
+ {pg_utf2wchar_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifier, 4}, /* PG_UTF8 */
+ {pg_mule2wchar_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifier, 4}, /* PG_MULE_INTERNAL */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN1 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN2 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN3 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN4 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN5 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN6 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN7 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN8 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN9 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN10 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1256 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1258 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN866 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN874 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_KOI8R */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1251 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1252 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-5 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-6 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-7 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-8 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1250 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1253 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1254 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1255 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1257 */
+ {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_KOI8U */
+ {0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2}, /* PG_SJIS */
+ {0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifier, 2}, /* PG_BIG5 */
+ {0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifier, 2}, /* PG_GBK */
+ {0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifier, 2}, /* PG_UHC */
+ {0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifier, 4}, /* PG_GB18030 */
+ {0, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifier, 3}, /* PG_JOHAB */
+ {0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2} /* PG_SHIFT_JIS_2004 */
+};
+
+/* returns the byte length of a word for mule internal code */
+int
+pg_mic_mblen(const unsigned char *mbstr)
+{
+ return pg_mule_mblen(mbstr);
+}
+
+/*
+ * Returns the byte length of a multibyte character.
+ */
+int
+pg_encoding_mblen(int encoding, const char *mbstr)
+{
+ Assert(PG_VALID_ENCODING(encoding));
+
+ return ((encoding >= 0 &&
+ encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
+ ((*pg_wchar_table[encoding].mblen) ((const unsigned char *) mbstr)) :
+ ((*pg_wchar_table[PG_SQL_ASCII].mblen) ((const unsigned char *) mbstr)));
+}
+
+/*
+ * Returns the display length of a multibyte character.
+ */
+int
+pg_encoding_dsplen(int encoding, const char *mbstr)
+{
+ Assert(PG_VALID_ENCODING(encoding));
+
+ return ((encoding >= 0 &&
+ encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
+ ((*pg_wchar_table[encoding].dsplen) ((const unsigned char *) mbstr)) :
+ ((*pg_wchar_table[PG_SQL_ASCII].dsplen) ((const unsigned char *) mbstr)));
+}
+
+/*
+ * Verify the first multibyte character of the given string.
+ * Return its byte length if good, -1 if bad. (See comments above for
+ * full details of the mbverify API.)
+ */
+int
+pg_encoding_verifymb(int encoding, const char *mbstr, int len)
+{
+ Assert(PG_VALID_ENCODING(encoding));
+
+ return ((encoding >= 0 &&
+ encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
+ ((*pg_wchar_table[encoding].mbverify) ((const unsigned char *) mbstr, len)) :
+ ((*pg_wchar_table[PG_SQL_ASCII].mbverify) ((const unsigned char *) mbstr, len)));
+}
+
+/*
+ * fetch maximum length of a given encoding
+ */
+int
+pg_encoding_max_length(int encoding)
+{
+ Assert(PG_VALID_ENCODING(encoding));
+
+ return pg_wchar_table[encoding].maxmblen;
+}
+
+#ifndef FRONTEND
+
+/*
+ * fetch maximum length of the encoding for the current database
+ */
+int
+pg_database_encoding_max_length(void)
+{
+ return pg_wchar_table[GetDatabaseEncoding()].maxmblen;
+}
+
+/*
+ * Verify mbstr to make sure that it is validly encoded in the current
+ * database encoding. Otherwise same as pg_verify_mbstr().
+ */
+bool
+pg_verifymbstr(const char *mbstr, int len, bool noError)
+{
+ return
+ pg_verify_mbstr_len(GetDatabaseEncoding(), mbstr, len, noError) >= 0;
+}
+
+/*
+ * Verify mbstr to make sure that it is validly encoded in the specified
+ * encoding.
+ */
+bool
+pg_verify_mbstr(int encoding, const char *mbstr, int len, bool noError)
+{
+ return pg_verify_mbstr_len(encoding, mbstr, len, noError) >= 0;
+}
+
+/*
+ * Verify mbstr to make sure that it is validly encoded in the specified
+ * encoding.
+ *
+ * mbstr is not necessarily zero terminated; length of mbstr is
+ * specified by len.
+ *
+ * If OK, return length of string in the encoding.
+ * If a problem is found, return -1 when noError is
+ * true; when noError is false, ereport() a descriptive message.
+ */
+int
+pg_verify_mbstr_len(int encoding, const char *mbstr, int len, bool noError)
+{
+ mbverifier mbverify;
+ int mb_len;
+
+ Assert(PG_VALID_ENCODING(encoding));
+
+ /*
+ * In single-byte encodings, we need only reject nulls (\0).
+ */
+ if (pg_encoding_max_length(encoding) <= 1)
+ {
+ const char *nullpos = memchr(mbstr, 0, len);
+
+ if (nullpos == NULL)
+ return len;
+ if (noError)
+ return -1;
+ report_invalid_encoding(encoding, nullpos, 1);
+ }
+
+ /* fetch function pointer just once */
+ mbverify = pg_wchar_table[encoding].mbverify;
+
+ mb_len = 0;
+
+ while (len > 0)
+ {
+ int l;
+
+ /* fast path for ASCII-subset characters */
+ if (!IS_HIGHBIT_SET(*mbstr))
+ {
+ if (*mbstr != '\0')
+ {
+ mb_len++;
+ mbstr++;
+ len--;
+ continue;
+ }
+ if (noError)
+ return -1;
+ report_invalid_encoding(encoding, mbstr, len);
+ }
+
+ l = (*mbverify) ((const unsigned char *) mbstr, len);
+
+ if (l < 0)
+ {
+ if (noError)
+ return -1;
+ report_invalid_encoding(encoding, mbstr, len);
+ }
+
+ mbstr += l;
+ len -= l;
+ mb_len++;
+ }
+ return mb_len;
+}
+
+/*
+ * check_encoding_conversion_args: check arguments of a conversion function
+ *
+ * "expected" arguments can be either an encoding ID or -1 to indicate that
+ * the caller will check whether it accepts the ID.
+ *
+ * Note: the errors here are not really user-facing, so elog instead of
+ * ereport seems sufficient. Also, we trust that the "expected" encoding
+ * arguments are valid encoding IDs, but we don't trust the actuals.
+ */
+void
+check_encoding_conversion_args(int src_encoding,
+ int dest_encoding,
+ int len,
+ int expected_src_encoding,
+ int expected_dest_encoding)
+{
+ if (!PG_VALID_ENCODING(src_encoding))
+ elog(ERROR, "invalid source encoding ID: %d", src_encoding);
+ if (src_encoding != expected_src_encoding && expected_src_encoding >= 0)
+ elog(ERROR, "expected source encoding \"%s\", but got \"%s\"",
+ pg_enc2name_tbl[expected_src_encoding].name,
+ pg_enc2name_tbl[src_encoding].name);
+ if (!PG_VALID_ENCODING(dest_encoding))
+ elog(ERROR, "invalid destination encoding ID: %d", dest_encoding);
+ if (dest_encoding != expected_dest_encoding && expected_dest_encoding >= 0)
+ elog(ERROR, "expected destination encoding \"%s\", but got \"%s\"",
+ pg_enc2name_tbl[expected_dest_encoding].name,
+ pg_enc2name_tbl[dest_encoding].name);
+ if (len < 0)
+ elog(ERROR, "encoding conversion length must not be negative");
+}
+
+/*
+ * report_invalid_encoding: complain about invalid multibyte character
+ *
+ * note: len is remaining length of string, not length of character;
+ * len must be greater than zero, as we always examine the first byte.
+ */
+void
+report_invalid_encoding(int encoding, const char *mbstr, int len)
+{
+ int l = pg_encoding_mblen(encoding, mbstr);
+ char buf[8 * 2 + 1];
+ char *p = buf;
+ int j,
+ jlimit;
+
+ jlimit = Min(l, len);
+ jlimit = Min(jlimit, 8); /* prevent buffer overrun */
+
+ for (j = 0; j < jlimit; j++)
+ p += sprintf(p, "%02x", (unsigned char) mbstr[j]);
+
+ ereport(ERROR,
+ (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
+ errmsg("invalid byte sequence for encoding \"%s\": 0x%s",
+ pg_enc2name_tbl[encoding].name,
+ buf)));
+}
+
+/*
+ * report_untranslatable_char: complain about untranslatable character
+ *
+ * note: len is remaining length of string, not length of character;
+ * len must be greater than zero, as we always examine the first byte.
+ */
+void
+report_untranslatable_char(int src_encoding, int dest_encoding,
+ const char *mbstr, int len)
+{
+ int l = pg_encoding_mblen(src_encoding, mbstr);
+ char buf[8 * 2 + 1];
+ char *p = buf;
+ int j,
+ jlimit;
+
+ jlimit = Min(l, len);
+ jlimit = Min(jlimit, 8); /* prevent buffer overrun */
+
+ for (j = 0; j < jlimit; j++)
+ p += sprintf(p, "%02x", (unsigned char) mbstr[j]);
+
+ ereport(ERROR,
+ (errcode(ERRCODE_UNTRANSLATABLE_CHARACTER),
+ errmsg("character 0x%s of encoding \"%s\" has no equivalent in \"%s\"",
+ buf,
+ pg_enc2name_tbl[src_encoding].name,
+ pg_enc2name_tbl[dest_encoding].name)));
+}
+
+#endif
[parent: 2f3cbd4e411f]