pg.ddx.io pgsql-patches@postgresql.org mailing list archive help / color / mirror / Atom feed From: Bill Huang <bill_huanghb@ybb.ne.jp>
To: Tatsuo Ishii <t-ishii@sra.co.jp>
Cc: pgsql-announce@postgresql.org
Cc: pgsql-patches@postgresql.org
Cc: bhuang@redhat.com
Cc: ynakai@redhat.com
Subject: Re: Chinese GB18030 support is implemented!
Date: Fri, 07 Jun 2002 14:08:23 +0900
Message-ID: <3D003FC7.7000801@ybb.ne.jp> (raw )
References: <3CFF0AE1.10205@ybb.ne.jp >
<20020607.102826.13767921.t-ishii@sra.co.jp >
Tatsuo Ishii wrote:
>>Hello,
>>
>>As postgresql is widely used in the world,many Chinese users are looking
>>forward to use such a high performanced database management
>>system.However since the Chinese new codepage standard GB18030 is not
>>completely supported,postgresql is limitted to be used in China.
>>
>>Now I have managed to implement the GB18030 support upon the latest
>>version,so the following functions are added after the patches are added.
>>
>>-Chinese GB18030 encoding is available on front-end side,while on
>>backend side,EUC_CN or MIC is used.
>>-Encoding convertion between MIC and GB18030 is implement.
>>-GB18030 locale support is available on front-end side.
>>-GB18030 locale test is added.
>>
>>Any help for testing with these patches and sugguestions for GB18030
>>support are greatly appreciated.
>>
>
>In your patches I found conversion functions between UTF-8 and GB18030
>using the same mapping for EUC_CN. I don't know the details of GB18030
>but I could guess that the mapping for EUC_CN might not work with
>GB18030. If so, what do you intend to add these function?
>--
>Tatsuo Ishii
>
>---------------------------(end of broadcast)---------------------------
>TIP 4: Don't 'kill -9' the postmaster
>
Now conversion between UTF-8 and GB18030 is implemented.
Regards,
Bill
--
/---------------------------/
Bill Huang)
E-mail:bill_huanghb@ybb.ne.jp
/---------------------------/
Attachments:
postgresql-7.2.1-conv-gb18030.patch (text/plain, 3.0K) download
--- postgresql-7.2.1/src/backend/utils/mb/conv.c.org Fri Jun 7 11:57:14 2002
+++ postgresql-7.2.1/src/backend/utils/mb/conv.c Fri Jun 7 14:15:26 2002
@@ -48,6 +48,8 @@
#include "Unicode/euc_jp_to_utf8.map"
#include "Unicode/utf8_to_euc_cn.map"
#include "Unicode/euc_cn_to_utf8.map"
+#include "Unicode/utf8_to_gb18030.map"
+#include "Unicode/gb18030_to_utf8.map"
#include "Unicode/utf8_to_euc_kr.map"
#include "Unicode/euc_kr_to_utf8.map"
#include "Unicode/utf8_to_euc_tw.map"
@@ -502,6 +504,96 @@
}
/*
+ * GB18030 ---> MIC
+ * Added by Bill Huang <bhuang@redhat.com>,<bill_huanghb@ybb.ne.jp>
+ */
+static void
+gb180302mic(unsigned char *gb18030, unsigned char *p, int len)
+{
+ int c1;
+ int c2;
+
+ while (len > 0 && (c1 = *gb18030++))
+ {
+ if (c1 < 0x80)
+ { /* should be ASCII */
+ len--;
+ *p++ = c1;
+ }
+ else if(c1 >= 0x81 && c1 <= 0xfe)
+ {
+ c2 = *gb18030++;
+
+ if(c2 >= 0x30 && c2 <= 0x69){
+ len -= 4;
+ *p++ = c1;
+ *p++ = c2;
+ *p++ = *gb18030++;
+ *p++ = *gb18030++;
+ *p++ = *gb18030++;
+ }
+ else if ((c2 >=0x40 && c2 <= 0x7e) ||(c2 >=0x80 && c2 <= 0xfe)){
+ len -= 2;
+ *p++ = c1;
+ *p++ = c2;
+ *p++ = *gb18030++;
+ }
+ else{ /*throw the strange code*/
+ len--;
+ }
+ }
+ }
+ *p = '\0';
+}
+
+/*
+ * MIC ---> GB18030
+ * Added by Bill Huang <bhuang@redhat.com>,<bill_huanghb@ybb.ne.jp>
+ */
+static void
+mic2gb18030(unsigned char *mic, unsigned char *p, int len)
+{
+ int c1;
+ int c2;
+
+ while (len > 0 && (c1 = *mic))
+ {
+ len -= pg_mic_mblen(mic++);
+
+ if (c1 <= 0x7f) /*ASCII*/
+ {
+ *p++ = c1;
+ }
+ else if (c1 >= 0x81 && c1 <= 0xfe)
+ {
+ c2 = *mic++;
+
+ if((c2 >= 0x40 && c2 <= 0x7e) || (c2 >= 0x80 && c2 <= 0xfe)){
+ *p++ = c1;
+ *p++ = c2;
+ }
+ else if(c2 >= 0x30 && c2 <= 0x39){
+ *p++ = c1;
+ *p++ = c2;
+ *p++ = *mic++;
+ *p++ = *mic++;
+ }
+ else{
+ mic--;
+ printBogusChar(&mic, &p);
+ mic--;
+ printBogusChar(&mic, &p);
+ }
+ }
+ else{
+ mic--;
+ printBogusChar(&mic, &p);
+ }
+ }
+ *p = '\0';
+}
+
+/*
* EUC_TW ---> MIC
*/
static void
@@ -1583,6 +1675,26 @@
}
/*
+ * UTF-8 ---> GB18030
+ */
+static void
+utf_to_gb18030(unsigned char *utf, unsigned char *euc, int len)
+
+{
+ utf_to_local(utf, euc, ULmapGB18030,
+ sizeof(ULmapGB18030) / sizeof(pg_utf_to_local), len);
+}
+
+/*
+ * GB18030 ---> UTF-8
+ */
+static void
+gb18030_to_utf(unsigned char *euc, unsigned char *utf, int len)
+{
+ local_to_utf(euc, utf, LUmapGB18030,
+ sizeof(LUmapGB18030) / sizeof(pg_local_to_utf), PG_GB18030, len);
+}
+/*
* UTF-8 ---> EUC_KR
*/
static void
@@ -1754,6 +1866,9 @@
PG_BIG5, big52mic, mic2big5, big5_to_utf, utf_to_big5
},
{
+ PG_GB18030, gb180302mic, mic2gb18030, gb18030_to_utf, utf_to_gb18030
+ },
+ {
PG_WIN1250, win12502mic, mic2win1250, 0, 0
},
};
@@ -1841,6 +1956,9 @@
PG_BIG5, big52mic, mic2big5, 0, 0
},
{
+ PG_GB18030, gb180302mic, mic2gb18030, 0, 0
+ },
+ {
PG_WIN1250, win12502mic, mic2win1250, 0, 0
},
};
UCS_to_GB18030.pl (text/plain, 2.0K) download
#! /usr/bin/perl
#
# Copyright 2002 by Bill Huang
#
# $Id: UCS_to_GB18030.pl,v 1.2 2002/06/06 06:10:18 Bill Huang Exp $
#
# Generate UTF-8 <--> GB18030 code conversion tables from
# map files provided by Unicode organization.
# Unfortunately it is prohibited by the organization
# to distribute the map files. So if you try to use this script,
# you have to obtain ISO10646-GB18030.TXT from
# the organization's ftp site.
#
# ISO10646-GB18030.TXT format:
# GB18030 code in hex
# UCS-2 code in hex
# # and Unicode name (not used in this script)
require "ucs2utf.pl";
# first generate UTF-8 --> GB18030 table
$in_file = "ISO10646-GB18030.TXT";
open( FILE, $in_file ) || die( "cannot open $in_file" );
while( <FILE> ){
chop;
if( /^#/ ){
next;
}
( $u, $c, $rest ) = split;
$utf = hex($u);
$code = hex($c);
$count++;
$array{ $utf } = ($code);
}
close( FILE );
#
# first, generate UTF8 --> GB18030 table
#
$file = "utf8_to_gb18030.map";
open( FILE, "> $file" ) || die( "cannot open $file" );
print FILE "static pg_utf_to_local ULmapGB18030[ $count ] = {\n";
for $index ( sort {$a <=> $b} keys( %array ) ){
$code = $array{ $index };
$count--;
if( $count == 0 ){
printf FILE " {0x%04x, 0x%04x}\n", $index, $code;
} else {
printf FILE " {0x%04x, 0x%04x},\n", $index, $code;
}
}
print FILE "};\n";
close(FILE);
#
# then generate GB18030 --> UTF8 table
#
reset 'array';
open( FILE, $in_file ) || die( "cannot open $in_file" );
while( <FILE> ){
chop;
if( /^#/ ){
next;
}
( $u, $c, $rest ) = split;
$utf = hex($u);
$code = hex($c);
$count++;
$array{ $code } = $utf;
}
close( FILE );
$file = "gb18030_to_utf8.map";
open( FILE, "> $file" ) || die( "cannot open $file" );
print FILE "static pg_local_to_utf LUmapGB18030[ $count ] = {\n";
for $index ( sort {$a <=> $b} keys( %array ) ){
$utf = $array{ $index };
$count--;
if( $count == 0 ){
printf FILE " {0x%04x, 0x%04x}\n", $index, $utf;
} else {
printf FILE " {0x%04x, 0x%04x},\n", $index, $utf;
}
}
print FILE "};\n";
close(FILE);
download
view thread (9+ messages) latest in thread
Message-ID: <3D003FC7.7000801@ybb.ne.jp>
Permalink: ../3D003FC7.7000801@ybb.ne.jp/
Also on: postgresql.org/message-id/3D003FC7.7000801@ybb.ne.jp
copy link · copy postgr.es · copy git am
reply Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-patches@postgresql.org
Cc: bill_huanghb@ybb.ne.jp, t-ishii@sra.co.jp, pgsql-announce@postgresql.org, bhuang@redhat.com, ynakai@redhat.com
Subject: Re: Chinese GB18030 support is implemented!
In-Reply-To: <3D003FC7.7000801@ybb.ne.jp>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox