Logo Search packages:      
Sourcecode: postgresql-8.4 version File versions

wchar.c

/*
 * conversion functions between pg_wchar and multibyte streams.
 * Tatsuo Ishii
 * $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.72 2009/03/02 21:18:43 tgl Exp $
 *
 */
/* can be used in either frontend or backend */
#ifdef FRONTEND
#include "postgres_fe.h"
#define Assert(condition)
#else
#include "postgres.h"
#endif

#include "mb/pg_wchar.h"


/*
 * conversion to pg_wchar is done by "table driven."
 * to add an encoding support, define mb2wchar_with_len(), mblen(), dsplen()
 * for the particular encoding. Note that if the encoding is only
 * supported in the client, you don't need to define
 * mb2wchar_with_len() function (SJIS is the case).
 *
 * These functions generally assume that their input is validly formed.
 * The "verifier" functions, further down in the file, have to be more
 * paranoid.  We expect that mblen() does not need to examine more than
 * the first byte of the character to discover the correct length.
 *
 * Note: for the display output of psql to work properly, the return values
 * of the dsplen functions must conform to the Unicode standard. In particular
 * the NUL character is zero width and control characters are generally
 * width -1. It is recommended that non-ASCII encodings refer their ASCII
 * subset to the ASCII routines to ensure consistency.
 */

/*
 * SQL/ASCII
 */
static int
pg_ascii2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
      int               cnt = 0;

      while (len > 0 && *from)
      {
            *to++ = *from++;
            len--;
            cnt++;
      }
      *to = 0;
      return cnt;
}

static int
pg_ascii_mblen(const unsigned char *s)
{
      return 1;
}

static int
pg_ascii_dsplen(const unsigned char *s)
{
      if (*s == '\0')
            return 0;
      if (*s < 0x20 || *s == 0x7f)
            return -1;

      return 1;
}

/*
 * EUC
 */
static int
pg_euc2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
      int               cnt = 0;

      while (len > 0 && *from)
      {
            if (*from == SS2 && len >= 2) /* JIS X 0201 (so called "1 byte
                                                             * KANA") */
            {
                  from++;
                  *to = (SS2 << 8) | *from++;
                  len -= 2;
            }
            else if (*from == SS3 && len >= 3)        /* JIS X 0212 KANJI */
            {
                  from++;
                  *to = (SS3 << 16) | (*from++ << 8);
                  *to |= *from++;
                  len -= 3;
            }
            else if (IS_HIGHBIT_SET(*from) && len >= 2)           /* JIS X 0208 KANJI */
            {
                  *to = *from++ << 8;
                  *to |= *from++;
                  len -= 2;
            }
            else
                  /* must be ASCII */
            {
                  *to = *from++;
                  len--;
            }
            to++;
            cnt++;
      }
      *to = 0;
      return cnt;
}

static inline int
pg_euc_mblen(const unsigned char *s)
{
      int               len;

      if (*s == SS2)
            len = 2;
      else if (*s == SS3)
            len = 3;
      else if (IS_HIGHBIT_SET(*s))
            len = 2;
      else
            len = 1;
      return len;
}

static inline int
pg_euc_dsplen(const unsigned char *s)
{
      int               len;

      if (*s == SS2)
            len = 2;
      else if (*s == SS3)
            len = 2;
      else if (IS_HIGHBIT_SET(*s))
            len = 2;
      else
            len = pg_ascii_dsplen(s);
      return len;
}

/*
 * EUC_JP
 */
static int
pg_eucjp2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
      return pg_euc2wchar_with_len(from, to, len);
}

static int
pg_eucjp_mblen(const unsigned char *s)
{
      return pg_euc_mblen(s);
}

static int
pg_eucjp_dsplen(const unsigned char *s)
{
      int               len;

      if (*s == SS2)
            len = 1;
      else if (*s == SS3)
            len = 2;
      else if (IS_HIGHBIT_SET(*s))
            len = 2;
      else
            len = pg_ascii_dsplen(s);
      return len;
}

/*
 * EUC_KR
 */
static int
pg_euckr2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
      return pg_euc2wchar_with_len(from, to, len);
}

static int
pg_euckr_mblen(const unsigned char *s)
{
      return pg_euc_mblen(s);
}

static int
pg_euckr_dsplen(const unsigned char *s)
{
      return pg_euc_dsplen(s);
}

/*
 * EUC_CN
 *
 */
static int
pg_euccn2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
      int               cnt = 0;

      while (len > 0 && *from)
      {
            if (*from == SS2 && len >= 3) /* code set 2 (unused?) */
            {
                  from++;
                  *to = (SS2 << 16) | (*from++ << 8);
                  *to |= *from++;
                  len -= 3;
            }
            else if (*from == SS3 && len >= 3)        /* code set 3 (unsed ?) */
            {
                  from++;
                  *to = (SS3 << 16) | (*from++ << 8);
                  *to |= *from++;
                  len -= 3;
            }
            else if (IS_HIGHBIT_SET(*from) && len >= 2)           /* code set 1 */
            {
                  *to = *from++ << 8;
                  *to |= *from++;
                  len -= 2;
            }
            else
            {
                  *to = *from++;
                  len--;
            }
            to++;
            cnt++;
      }
      *to = 0;
      return cnt;
}

static int
pg_euccn_mblen(const unsigned char *s)
{
      int               len;

      if (IS_HIGHBIT_SET(*s))
            len = 2;
      else
            len = 1;
      return len;
}

static int
pg_euccn_dsplen(const unsigned char *s)
{
      int               len;

      if (IS_HIGHBIT_SET(*s))
            len = 2;
      else
            len = pg_ascii_dsplen(s);
      return len;
}

/*
 * EUC_TW
 *
 */
static int
pg_euctw2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
      int               cnt = 0;

      while (len > 0 && *from)
      {
            if (*from == SS2 && len >= 4) /* code set 2 */
            {
                  from++;
                  *to = (((uint32) SS2) << 24) | (*from++ << 16);
                  *to |= *from++ << 8;
                  *to |= *from++;
                  len -= 4;
            }
            else if (*from == SS3 && len >= 3)        /* code set 3 (unused?) */
            {
                  from++;
                  *to = (SS3 << 16) | (*from++ << 8);
                  *to |= *from++;
                  len -= 3;
            }
            else if (IS_HIGHBIT_SET(*from) && len >= 2)           /* code set 2 */
            {
                  *to = *from++ << 8;
                  *to |= *from++;
                  len -= 2;
            }
            else
            {
                  *to = *from++;
                  len--;
            }
            to++;
            cnt++;
      }
      *to = 0;
      return cnt;
}

static int
pg_euctw_mblen(const unsigned char *s)
{
      int               len;

      if (*s == SS2)
            len = 4;
      else if (*s == SS3)
            len = 3;
      else if (IS_HIGHBIT_SET(*s))
            len = 2;
      else
            len = 1;
      return len;
}

static int
pg_euctw_dsplen(const unsigned char *s)
{
      int               len;

      if (*s == SS2)
            len = 2;
      else if (*s == SS3)
            len = 2;
      else if (IS_HIGHBIT_SET(*s))
            len = 2;
      else
            len = pg_ascii_dsplen(s);
      return len;
}

/*
 * JOHAB
 */
static int
pg_johab_mblen(const unsigned char *s)
{
      return pg_euc_mblen(s);
}

static int
pg_johab_dsplen(const unsigned char *s)
{
      return pg_euc_dsplen(s);
}

/*
 * convert UTF8 string to pg_wchar (UCS-4)
 * caller must allocate enough space for "to", including a trailing zero!
 * len: length of from.
 * "from" not necessarily null terminated.
 */
static int
pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
      int               cnt = 0;
      uint32            c1,
                        c2,
                        c3,
                        c4;

      while (len > 0 && *from)
      {
            if ((*from & 0x80) == 0)
            {
                  *to = *from++;
                  len--;
            }
            else if ((*from & 0xe0) == 0xc0)
            {
                  if (len < 2)
                        break;                  /* drop trailing incomplete char */
                  c1 = *from++ & 0x1f;
                  c2 = *from++ & 0x3f;
                  *to = (c1 << 6) | c2;
                  len -= 2;
            }
            else if ((*from & 0xf0) == 0xe0)
            {
                  if (len < 3)
                        break;                  /* drop trailing incomplete char */
                  c1 = *from++ & 0x0f;
                  c2 = *from++ & 0x3f;
                  c3 = *from++ & 0x3f;
                  *to = (c1 << 12) | (c2 << 6) | c3;
                  len -= 3;
            }
            else if ((*from & 0xf8) == 0xf0)
            {
                  if (len < 4)
                        break;                  /* drop trailing incomplete char */
                  c1 = *from++ & 0x07;
                  c2 = *from++ & 0x3f;
                  c3 = *from++ & 0x3f;
                  c4 = *from++ & 0x3f;
                  *to = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
                  len -= 4;
            }
            else
            {
                  /* treat a bogus char as length 1; not ours to raise error */
                  *to = *from++;
                  len--;
            }
            to++;
            cnt++;
      }
      *to = 0;
      return cnt;
}


/*
 * Map a Unicode code point to UTF-8.  utf8string must have 4 bytes of
 * space allocated.
 */
unsigned char *
unicode_to_utf8(pg_wchar c, unsigned char *utf8string)
{
      if (c <= 0x7F)
      {
            utf8string[0] = c;
      }
      else if (c <= 0x7FF)
      {
            utf8string[0] = 0xC0 | ((c >> 6) & 0x1F);
            utf8string[1] = 0x80 | (c & 0x3F);
      }
      else if (c <= 0xFFFF)
      {
            utf8string[0] = 0xE0 | ((c >> 12) & 0x0F);
            utf8string[1] = 0x80 | ((c >> 6) & 0x3F);
            utf8string[2] = 0x80 | (c & 0x3F);
      }
      else
      {
            utf8string[0] = 0xF0 | ((c >> 18) & 0x07);
            utf8string[1] = 0x80 | ((c >> 12) & 0x3F);
            utf8string[2] = 0x80 | ((c >> 6) & 0x3F);
            utf8string[3] = 0x80 | (c & 0x3F);
      }

      return utf8string;
}


/*
 * Return the byte length of a UTF8 character pointed to by s
 *
 * Note: in the current implementation we do not support UTF8 sequences
 * of more than 4 bytes; hence do NOT return a value larger than 4.
 * We return "1" for any leading byte that is either flat-out illegal or
 * indicates a length larger than we support.
 *
 * pg_utf2wchar_with_len(), utf2ucs(), pg_utf8_islegal(), and perhaps
 * other places would need to be fixed to change this.
 */
int
pg_utf_mblen(const unsigned char *s)
{
      int               len;

      if ((*s & 0x80) == 0)
            len = 1;
      else if ((*s & 0xe0) == 0xc0)
            len = 2;
      else if ((*s & 0xf0) == 0xe0)
            len = 3;
      else if ((*s & 0xf8) == 0xf0)
            len = 4;
#ifdef NOT_USED
      else if ((*s & 0xfc) == 0xf8)
            len = 5;
      else if ((*s & 0xfe) == 0xfc)
            len = 6;
#endif
      else
            len = 1;
      return len;
}

/*
 * This is an implementation of wcwidth() and wcswidth() as defined in
 * "The Single UNIX Specification, Version 2, The Open Group, 1997"
 * <http://www.UNIX-systems.org/online.html>
 *
 * Markus Kuhn -- 2001-09-08 -- public domain
 *
 * customised for PostgreSQL
 *
 * original available at : http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
 */

struct mbinterval
{
      unsigned short first;
      unsigned short last;
};

/* auxiliary function for binary search in interval table */
static int
mbbisearch(pg_wchar ucs, const struct mbinterval * table, int max)
{
      int               min = 0;
      int               mid;

      if (ucs < table[0].first || ucs > table[max].last)
            return 0;
      while (max >= min)
      {
            mid = (min + max) / 2;
            if (ucs > table[mid].last)
                  min = mid + 1;
            else if (ucs < table[mid].first)
                  max = mid - 1;
            else
                  return 1;
      }

      return 0;
}


/* The following functions define the column width of an ISO 10646
 * character as follows:
 *
 *      - The null character (U+0000) has a column width of 0.
 *
 *      - Other C0/C1 control characters and DEL will lead to a return
 *          value of -1.
 *
 *      - Non-spacing and enclosing combining characters (general
 *          category code Mn or Me in the Unicode database) have a
 *          column width of 0.
 *
 *      - Other format characters (general category code Cf in the Unicode
 *          database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
 *
 *      - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
 *          have a column width of 0.
 *
 *      - Spacing characters in the East Asian Wide (W) or East Asian
 *          FullWidth (F) category as defined in Unicode Technical
 *          Report #11 have a column width of 2.
 *
 *      - All remaining characters (including all printable
 *          ISO 8859-1 and WGL4 characters, Unicode control characters,
 *          etc.) have a column width of 1.
 *
 * This implementation assumes that wchar_t characters are encoded
 * in ISO 10646.
 */

static int
ucs_wcwidth(pg_wchar ucs)
{
      /* sorted list of non-overlapping intervals of non-spacing characters */
      static const struct mbinterval combining[] = {
            {0x0300, 0x034E}, {0x0360, 0x0362}, {0x0483, 0x0486},
            {0x0488, 0x0489}, {0x0591, 0x05A1}, {0x05A3, 0x05B9},
            {0x05BB, 0x05BD}, {0x05BF, 0x05BF}, {0x05C1, 0x05C2},
            {0x05C4, 0x05C4}, {0x064B, 0x0655}, {0x0670, 0x0670},
            {0x06D6, 0x06E4}, {0x06E7, 0x06E8}, {0x06EA, 0x06ED},
            {0x070F, 0x070F}, {0x0711, 0x0711}, {0x0730, 0x074A},
            {0x07A6, 0x07B0}, {0x0901, 0x0902}, {0x093C, 0x093C},
            {0x0941, 0x0948}, {0x094D, 0x094D}, {0x0951, 0x0954},
            {0x0962, 0x0963}, {0x0981, 0x0981}, {0x09BC, 0x09BC},
            {0x09C1, 0x09C4}, {0x09CD, 0x09CD}, {0x09E2, 0x09E3},
            {0x0A02, 0x0A02}, {0x0A3C, 0x0A3C}, {0x0A41, 0x0A42},
            {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A70, 0x0A71},
            {0x0A81, 0x0A82}, {0x0ABC, 0x0ABC}, {0x0AC1, 0x0AC5},
            {0x0AC7, 0x0AC8}, {0x0ACD, 0x0ACD}, {0x0B01, 0x0B01},
            {0x0B3C, 0x0B3C}, {0x0B3F, 0x0B3F}, {0x0B41, 0x0B43},
            {0x0B4D, 0x0B4D}, {0x0B56, 0x0B56}, {0x0B82, 0x0B82},
            {0x0BC0, 0x0BC0}, {0x0BCD, 0x0BCD}, {0x0C3E, 0x0C40},
            {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56},
            {0x0CBF, 0x0CBF}, {0x0CC6, 0x0CC6}, {0x0CCC, 0x0CCD},
            {0x0D41, 0x0D43}, {0x0D4D, 0x0D4D}, {0x0DCA, 0x0DCA},
            {0x0DD2, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0E31, 0x0E31},
            {0x0E34, 0x0E3A}, {0x0E47, 0x0E4E}, {0x0EB1, 0x0EB1},
            {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC}, {0x0EC8, 0x0ECD},
            {0x0F18, 0x0F19}, {0x0F35, 0x0F35}, {0x0F37, 0x0F37},
            {0x0F39, 0x0F39}, {0x0F71, 0x0F7E}, {0x0F80, 0x0F84},
            {0x0F86, 0x0F87}, {0x0F90, 0x0F97}, {0x0F99, 0x0FBC},
            {0x0FC6, 0x0FC6}, {0x102D, 0x1030}, {0x1032, 0x1032},
            {0x1036, 0x1037}, {0x1039, 0x1039}, {0x1058, 0x1059},
            {0x1160, 0x11FF}, {0x17B7, 0x17BD}, {0x17C6, 0x17C6},
            {0x17C9, 0x17D3}, {0x180B, 0x180E}, {0x18A9, 0x18A9},
            {0x200B, 0x200F}, {0x202A, 0x202E}, {0x206A, 0x206F},
            {0x20D0, 0x20E3}, {0x302A, 0x302F}, {0x3099, 0x309A},
            {0xFB1E, 0xFB1E}, {0xFE20, 0xFE23}, {0xFEFF, 0xFEFF},
            {0xFFF9, 0xFFFB}
      };

      /* test for 8-bit control characters */
      if (ucs == 0)
            return 0;

      if (ucs < 0x20 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff)
            return -1;

      /* binary search in table of non-spacing characters */
      if (mbbisearch(ucs, combining,
                           sizeof(combining) / sizeof(struct mbinterval) - 1))
            return 0;

      /*
       * if we arrive here, ucs is not a combining or C0/C1 control character
       */

      return 1 +
            (ucs >= 0x1100 &&
             (ucs <= 0x115f ||            /* Hangul Jamo init. consonants */
              (ucs >= 0x2e80 && ucs <= 0xa4cf && (ucs & ~0x0011) != 0x300a &&
               ucs != 0x303f) ||    /* CJK ... Yi */
              (ucs >= 0xac00 && ucs <= 0xd7a3) ||     /* Hangul Syllables */
              (ucs >= 0xf900 && ucs <= 0xfaff) ||     /* CJK Compatibility
                                                                         * Ideographs */
              (ucs >= 0xfe30 && ucs <= 0xfe6f) ||     /* CJK Compatibility Forms */
              (ucs >= 0xff00 && ucs <= 0xff5f) ||     /* Fullwidth Forms */
              (ucs >= 0xffe0 && ucs <= 0xffe6) ||
              (ucs >= 0x20000 && ucs <= 0x2ffff)));
}

static pg_wchar
utf2ucs(const unsigned char *c)
{
      /*
       * one char version of pg_utf2wchar_with_len. no control here, c must
       * point to a large enough string
       */
      if ((*c & 0x80) == 0)
            return (pg_wchar) c[0];
      else if ((*c & 0xe0) == 0xc0)
            return (pg_wchar) (((c[0] & 0x1f) << 6) |
                                       (c[1] & 0x3f));
      else if ((*c & 0xf0) == 0xe0)
            return (pg_wchar) (((c[0] & 0x0f) << 12) |
                                       ((c[1] & 0x3f) << 6) |
                                       (c[2] & 0x3f));
      else if ((*c & 0xf8) == 0xf0)
            return (pg_wchar) (((c[0] & 0x07) << 18) |
                                       ((c[1] & 0x3f) << 12) |
                                       ((c[2] & 0x3f) << 6) |
                                       (c[3] & 0x3f));
      else
            /* that is an invalid code on purpose */
            return 0xffffffff;
}

static int
pg_utf_dsplen(const unsigned char *s)
{
      return ucs_wcwidth(utf2ucs(s));
}

/*
 * convert mule internal code to pg_wchar
 * caller should allocate enough space for "to"
 * len: length of from.
 * "from" not necessarily null terminated.
 */
static int
pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
      int               cnt = 0;

      while (len > 0 && *from)
      {
            if (IS_LC1(*from) && len >= 2)
            {
                  *to = *from++ << 16;
                  *to |= *from++;
                  len -= 2;
            }
            else if (IS_LCPRV1(*from) && len >= 3)
            {
                  from++;
                  *to = *from++ << 16;
                  *to |= *from++;
                  len -= 3;
            }
            else if (IS_LC2(*from) && len >= 3)
            {
                  *to = *from++ << 16;
                  *to |= *from++ << 8;
                  *to |= *from++;
                  len -= 3;
            }
            else if (IS_LCPRV2(*from) && len >= 4)
            {
                  from++;
                  *to = *from++ << 16;
                  *to |= *from++ << 8;
                  *to |= *from++;
                  len -= 4;
            }
            else
            {                                   /* assume ASCII */
                  *to = (unsigned char) *from++;
                  len--;
            }
            to++;
            cnt++;
      }
      *to = 0;
      return cnt;
}

int
pg_mule_mblen(const unsigned char *s)
{
      int               len;

      if (IS_LC1(*s))
            len = 2;
      else if (IS_LCPRV1(*s))
            len = 3;
      else if (IS_LC2(*s))
            len = 3;
      else if (IS_LCPRV2(*s))
            len = 4;
      else
            len = 1;                      /* assume ASCII */
      return len;
}

static int
pg_mule_dsplen(const unsigned char *s)
{
      int               len;

      if (IS_LC1(*s))
            len = 1;
      else if (IS_LCPRV1(*s))
            len = 1;
      else if (IS_LC2(*s))
            len = 2;
      else if (IS_LCPRV2(*s))
            len = 2;
      else
            len = 1;                      /* assume ASCII */

      return len;
}

/*
 * ISO8859-1
 */
static int
pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
      int               cnt = 0;

      while (len > 0 && *from)
      {
            *to++ = *from++;
            len--;
            cnt++;
      }
      *to = 0;
      return cnt;
}

static int
pg_latin1_mblen(const unsigned char *s)
{
      return 1;
}

static int
pg_latin1_dsplen(const unsigned char *s)
{
      return pg_ascii_dsplen(s);
}

/*
 * SJIS
 */
static int
pg_sjis_mblen(const unsigned char *s)
{
      int               len;

      if (*s >= 0xa1 && *s <= 0xdf)
            len = 1;                      /* 1 byte kana? */
      else if (IS_HIGHBIT_SET(*s))
            len = 2;                      /* kanji? */
      else
            len = 1;                      /* should be ASCII */
      return len;
}

static int
pg_sjis_dsplen(const unsigned char *s)
{
      int               len;

      if (*s >= 0xa1 && *s <= 0xdf)
            len = 1;                      /* 1 byte kana? */
      else if (IS_HIGHBIT_SET(*s))
            len = 2;                      /* kanji? */
      else
            len = pg_ascii_dsplen(s);           /* should be ASCII */
      return len;
}

/*
 * Big5
 */
static int
pg_big5_mblen(const unsigned char *s)
{
      int               len;

      if (IS_HIGHBIT_SET(*s))
            len = 2;                      /* kanji? */
      else
            len = 1;                      /* should be ASCII */
      return len;
}

static int
pg_big5_dsplen(const unsigned char *s)
{
      int               len;

      if (IS_HIGHBIT_SET(*s))
            len = 2;                      /* kanji? */
      else
            len = pg_ascii_dsplen(s);           /* should be ASCII */
      return len;
}

/*
 * GBK
 */
static int
pg_gbk_mblen(const unsigned char *s)
{
      int               len;

      if (IS_HIGHBIT_SET(*s))
            len = 2;                      /* kanji? */
      else
            len = 1;                      /* should be ASCII */
      return len;
}

static int
pg_gbk_dsplen(const unsigned char *s)
{
      int               len;

      if (IS_HIGHBIT_SET(*s))
            len = 2;                      /* kanji? */
      else
            len = pg_ascii_dsplen(s);           /* should be ASCII */
      return len;
}

/*
 * UHC
 */
static int
pg_uhc_mblen(const unsigned char *s)
{
      int               len;

      if (IS_HIGHBIT_SET(*s))
            len = 2;                      /* 2byte? */
      else
            len = 1;                      /* should be ASCII */
      return len;
}

static int
pg_uhc_dsplen(const unsigned char *s)
{
      int               len;

      if (IS_HIGHBIT_SET(*s))
            len = 2;                      /* 2byte? */
      else
            len = pg_ascii_dsplen(s);           /* should be ASCII */
      return len;
}

/*
 *    * GB18030
 *     * Added by Bill Huang <bhuang@redhat.com>,<bill_huanghb@ybb.ne.jp>
 *      */
static int
pg_gb18030_mblen(const unsigned char *s)
{
      int               len;

      if (!IS_HIGHBIT_SET(*s))
            len = 1;                      /* ASCII */
      else
      {
            if ((*(s + 1) >= 0x40 && *(s + 1) <= 0x7e) || (*(s + 1) >= 0x80 && *(s + 1) <= 0xfe))
                  len = 2;
            else if (*(s + 1) >= 0x30 && *(s + 1) <= 0x39)
                  len = 4;
            else
                  len = 2;
      }
      return len;
}

static int
pg_gb18030_dsplen(const unsigned char *s)
{
      int               len;

      if (IS_HIGHBIT_SET(*s))
            len = 2;
      else
            len = pg_ascii_dsplen(s);           /* ASCII */
      return len;
}

/*
 *-------------------------------------------------------------------
 * multibyte sequence validators
 *
 * These functions accept "s", a pointer to the first byte of a string,
 * and "len", the remaining length of the string.  If there is a validly
 * encoded character beginning at *s, return its length in bytes; else
 * return -1.
 *
 * The functions can assume that len > 0 and that *s != '\0', but they must
 * test for and reject zeroes in any additional bytes of a multibyte character.
 *
 * Note that this definition allows the function for a single-byte
 * encoding to be just "return 1".
 *-------------------------------------------------------------------
 */

static int
pg_ascii_verifier(const unsigned char *s, int len)
{
      return 1;
}

#define IS_EUC_RANGE_VALID(c) ((c) >= 0xa1 && (c) <= 0xfe)

static int
pg_eucjp_verifier(const unsigned char *s, int len)
{
      int               l;
      unsigned char c1,
                        c2;

      c1 = *s++;

      switch (c1)
      {
            case SS2:                     /* JIS X 0201 */
                  l = 2;
                  if (l > len)
                        return -1;
                  c2 = *s++;
                  if (c2 < 0xa1 || c2 > 0xdf)
                        return -1;
                  break;

            case SS3:                     /* JIS X 0212 */
                  l = 3;
                  if (l > len)
                        return -1;
                  c2 = *s++;
                  if (!IS_EUC_RANGE_VALID(c2))
                        return -1;
                  c2 = *s++;
                  if (!IS_EUC_RANGE_VALID(c2))
                        return -1;
                  break;

            default:
                  if (IS_HIGHBIT_SET(c1))       /* JIS X 0208? */
                  {
                        l = 2;
                        if (l > len)
                              return -1;
                        if (!IS_EUC_RANGE_VALID(c1))
                              return -1;
                        c2 = *s++;
                        if (!IS_EUC_RANGE_VALID(c2))
                              return -1;
                  }
                  else
                        /* must be ASCII */
                  {
                        l = 1;
                  }
                  break;
      }

      return l;
}

static int
pg_euckr_verifier(const unsigned char *s, int len)
{
      int               l;
      unsigned char c1,
                        c2;

      c1 = *s++;

      if (IS_HIGHBIT_SET(c1))
      {
            l = 2;
            if (l > len)
                  return -1;
            if (!IS_EUC_RANGE_VALID(c1))
                  return -1;
            c2 = *s++;
            if (!IS_EUC_RANGE_VALID(c2))
                  return -1;
      }
      else
            /* must be ASCII */
      {
            l = 1;
      }

      return l;
}

/* EUC-CN byte sequences are exactly same as EUC-KR */
#define pg_euccn_verifier     pg_euckr_verifier

static int
pg_euctw_verifier(const unsigned char *s, int len)
{
      int               l;
      unsigned char c1,
                        c2;

      c1 = *s++;

      switch (c1)
      {
            case SS2:                     /* CNS 11643 Plane 1-7 */
                  l = 4;
                  if (l > len)
                        return -1;
                  c2 = *s++;
                  if (c2 < 0xa1 || c2 > 0xa7)
                        return -1;
                  c2 = *s++;
                  if (!IS_EUC_RANGE_VALID(c2))
                        return -1;
                  c2 = *s++;
                  if (!IS_EUC_RANGE_VALID(c2))
                        return -1;
                  break;

            case SS3:                     /* unused */
                  return -1;

            default:
                  if (IS_HIGHBIT_SET(c1))       /* CNS 11643 Plane 1 */
                  {
                        l = 2;
                        if (l > len)
                              return -1;
                        /* no further range check on c1? */
                        c2 = *s++;
                        if (!IS_EUC_RANGE_VALID(c2))
                              return -1;
                  }
                  else
                        /* must be ASCII */
                  {
                        l = 1;
                  }
                  break;
      }
      return l;
}

static int
pg_johab_verifier(const unsigned char *s, int len)
{
      int               l,
                        mbl;
      unsigned char c;

      l = mbl = pg_johab_mblen(s);

      if (len < l)
            return -1;

      if (!IS_HIGHBIT_SET(*s))
            return mbl;

      while (--l > 0)
      {
            c = *++s;
            if (!IS_EUC_RANGE_VALID(c))
                  return -1;
      }
      return mbl;
}

static int
pg_mule_verifier(const unsigned char *s, int len)
{
      int               l,
                        mbl;
      unsigned char c;

      l = mbl = pg_mule_mblen(s);

      if (len < l)
            return -1;

      while (--l > 0)
      {
            c = *++s;
            if (!IS_HIGHBIT_SET(c))
                  return -1;
      }
      return mbl;
}

static int
pg_latin1_verifier(const unsigned char *s, int len)
{
      return 1;
}

static int
pg_sjis_verifier(const unsigned char *s, int len)
{
      int               l,
                        mbl;
      unsigned char c1,
                        c2;

      l = mbl = pg_sjis_mblen(s);

      if (len < l)
            return -1;

      if (l == 1)                         /* pg_sjis_mblen already verified it */
            return mbl;

      c1 = *s++;
      c2 = *s;
      if (!ISSJISHEAD(c1) || !ISSJISTAIL(c2))
            return -1;
      return mbl;
}

static int
pg_big5_verifier(const unsigned char *s, int len)
{
      int               l,
                        mbl;

      l = mbl = pg_big5_mblen(s);

      if (len < l)
            return -1;

      while (--l > 0)
      {
            if (*++s == '\0')
                  return -1;
      }

      return mbl;
}

static int
pg_gbk_verifier(const unsigned char *s, int len)
{
      int               l,
                        mbl;

      l = mbl = pg_gbk_mblen(s);

      if (len < l)
            return -1;

      while (--l > 0)
      {
            if (*++s == '\0')
                  return -1;
      }

      return mbl;
}

static int
pg_uhc_verifier(const unsigned char *s, int len)
{
      int               l,
                        mbl;

      l = mbl = pg_uhc_mblen(s);

      if (len < l)
            return -1;

      while (--l > 0)
      {
            if (*++s == '\0')
                  return -1;
      }

      return mbl;
}

static int
pg_gb18030_verifier(const unsigned char *s, int len)
{
      int               l,
                        mbl;

      l = mbl = pg_gb18030_mblen(s);

      if (len < l)
            return -1;

      while (--l > 0)
      {
            if (*++s == '\0')
                  return -1;
      }

      return mbl;
}

static int
pg_utf8_verifier(const unsigned char *s, int len)
{
      int               l = pg_utf_mblen(s);

      if (len < l)
            return -1;

      if (!pg_utf8_islegal(s, l))
            return -1;

      return l;
}

/*
 * Check for validity of a single UTF-8 encoded character
 *
 * This directly implements the rules in RFC3629.  The bizarre-looking
 * restrictions on the second byte are meant to ensure that there isn't
 * more than one encoding of a given Unicode character point; that is,
 * you may not use a longer-than-necessary byte sequence with high order
 * zero bits to represent a character that would fit in fewer bytes.
 * To do otherwise is to create security hazards (eg, create an apparent
 * non-ASCII character that decodes to plain ASCII).
 *
 * length is assumed to have been obtained by pg_utf_mblen(), and the
 * caller must have checked that that many bytes are present in the buffer.
 */
bool
pg_utf8_islegal(const unsigned char *source, int length)
{
      unsigned char a;

      switch (length)
      {
            default:
                  /* reject lengths 5 and 6 for now */
                  return false;
            case 4:
                  a = source[3];
                  if (a < 0x80 || a > 0xBF)
                        return false;
                  /* FALL THRU */
            case 3:
                  a = source[2];
                  if (a < 0x80 || a > 0xBF)
                        return false;
                  /* FALL THRU */
            case 2:
                  a = source[1];
                  switch (*source)
                  {
                        case 0xE0:
                              if (a < 0xA0 || a > 0xBF)
                                    return false;
                              break;
                        case 0xED:
                              if (a < 0x80 || a > 0x9F)
                                    return false;
                              break;
                        case 0xF0:
                              if (a < 0x90 || a > 0xBF)
                                    return false;
                              break;
                        case 0xF4:
                              if (a < 0x80 || a > 0x8F)
                                    return false;
                              break;
                        default:
                              if (a < 0x80 || a > 0xBF)
                                    return false;
                              break;
                  }
                  /* FALL THRU */
            case 1:
                  a = *source;
                  if (a >= 0x80 && a < 0xC2)
                        return false;
                  if (a > 0xF4)
                        return false;
                  break;
      }
      return true;
}

/*
 *-------------------------------------------------------------------
 * encoding info table
 * XXX must be sorted by the same order as enum pg_enc (in mb/pg_wchar.h)
 *-------------------------------------------------------------------
 */
pg_wchar_tbl pg_wchar_table[] = {
      {pg_ascii2wchar_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifier, 1},   /* PG_SQL_ASCII   */
      {pg_eucjp2wchar_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3},   /* PG_EUC_JP */
      {pg_euccn2wchar_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifier, 2},   /* PG_EUC_CN */
      {pg_euckr2wchar_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifier, 3},   /* PG_EUC_KR */
      {pg_euctw2wchar_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifier, 4},   /* PG_EUC_TW */
      {pg_eucjp2wchar_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3},   /* PG_EUC_JIS_2004 */
      {pg_utf2wchar_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifier, 4},    /* PG_UTF8 */
      {pg_mule2wchar_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifier, 4},       /* PG_MULE_INTERNAL */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN1 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN2 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN3 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN4 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN5 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN6 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN7 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN8 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN9 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN10 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1256 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1258 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN866 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN874 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_KOI8R */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1251 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1252 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* ISO-8859-5 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* ISO-8859-6 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* ISO-8859-7 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* ISO-8859-8 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1250 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1253 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1254 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1255 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1257 */
      {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_KOI8U */
      {0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2},    /* PG_SJIS */
      {0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifier, 2},    /* PG_BIG5 */
      {0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifier, 2},       /* PG_GBK */
      {0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifier, 2},       /* PG_UHC */
      {0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifier, 4}, /* PG_GB18030 */
      {0, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifier, 3}, /* PG_JOHAB */
      {0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2}           /* PG_SHIFT_JIS_2004 */
};

/* returns the byte length of a word for mule internal code */
int
pg_mic_mblen(const unsigned char *mbstr)
{
      return pg_mule_mblen(mbstr);
}

/*
 * Returns the byte length of a multibyte character.
 */
int
pg_encoding_mblen(int encoding, const char *mbstr)
{
      Assert(PG_VALID_ENCODING(encoding));

      return ((encoding >= 0 &&
                   encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
            ((*pg_wchar_table[encoding].mblen) ((const unsigned char *) mbstr)) :
      ((*pg_wchar_table[PG_SQL_ASCII].mblen) ((const unsigned char *) mbstr)));
}

/*
 * Returns the display length of a multibyte character.
 */
int
pg_encoding_dsplen(int encoding, const char *mbstr)
{
      Assert(PG_VALID_ENCODING(encoding));

      return ((encoding >= 0 &&
                   encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
         ((*pg_wchar_table[encoding].dsplen) ((const unsigned char *) mbstr)) :
      ((*pg_wchar_table[PG_SQL_ASCII].dsplen) ((const unsigned char *) mbstr)));
}

/*
 * Verify the first multibyte character of the given string.
 * Return its byte length if good, -1 if bad.  (See comments above for
 * full details of the mbverify API.)
 */
int
pg_encoding_verifymb(int encoding, const char *mbstr, int len)
{
      Assert(PG_VALID_ENCODING(encoding));

      return ((encoding >= 0 &&
                   encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
                  ((*pg_wchar_table[encoding].mbverify) ((const unsigned char *) mbstr, len)) :
                  ((*pg_wchar_table[PG_SQL_ASCII].mbverify) ((const unsigned char *) mbstr, len)));
}

/*
 * fetch maximum length of a given encoding
 */
int
pg_encoding_max_length(int encoding)
{
      Assert(PG_VALID_ENCODING(encoding));

      return pg_wchar_table[encoding].maxmblen;
}

#ifndef FRONTEND

/*
 * fetch maximum length of the encoding for the current database
 */
int
pg_database_encoding_max_length(void)
{
      return pg_wchar_table[GetDatabaseEncoding()].maxmblen;
}

/*
 * Verify mbstr to make sure that it is validly encoded in the current
 * database encoding.  Otherwise same as pg_verify_mbstr().
 */
bool
pg_verifymbstr(const char *mbstr, int len, bool noError)
{
      return
            pg_verify_mbstr_len(GetDatabaseEncoding(), mbstr, len, noError) >= 0;
}

/*
 * Verify mbstr to make sure that it is validly encoded in the specified
 * encoding.
 */
bool
pg_verify_mbstr(int encoding, const char *mbstr, int len, bool noError)
{
      return pg_verify_mbstr_len(encoding, mbstr, len, noError) >= 0;
}

/*
 * Verify mbstr to make sure that it is validly encoded in the specified
 * encoding.
 *
 * mbstr is not necessarily zero terminated; length of mbstr is
 * specified by len.
 *
 * If OK, return length of string in the encoding.
 * If a problem is found, return -1 when noError is
 * true; when noError is false, ereport() a descriptive message.
 */
int
pg_verify_mbstr_len(int encoding, const char *mbstr, int len, bool noError)
{
      mbverifier  mbverify;
      int               mb_len;

      Assert(PG_VALID_ENCODING(encoding));

      /*
       * In single-byte encodings, we need only reject nulls (\0).
       */
      if (pg_encoding_max_length(encoding) <= 1)
      {
            const char *nullpos = memchr(mbstr, 0, len);

            if (nullpos == NULL)
                  return len;
            if (noError)
                  return -1;
            report_invalid_encoding(encoding, nullpos, 1);
      }

      /* fetch function pointer just once */
      mbverify = pg_wchar_table[encoding].mbverify;

      mb_len = 0;

      while (len > 0)
      {
            int               l;

            /* fast path for ASCII-subset characters */
            if (!IS_HIGHBIT_SET(*mbstr))
            {
                  if (*mbstr != '\0')
                  {
                        mb_len++;
                        mbstr++;
                        len--;
                        continue;
                  }
                  if (noError)
                        return -1;
                  report_invalid_encoding(encoding, mbstr, len);
            }

            l = (*mbverify) ((const unsigned char *) mbstr, len);

            if (l < 0)
            {
                  if (noError)
                        return -1;
                  report_invalid_encoding(encoding, mbstr, len);
            }

            mbstr += l;
            len -= l;
            mb_len++;
      }
      return mb_len;
}

/*
 * check_encoding_conversion_args: check arguments of a conversion function
 *
 * "expected" arguments can be either an encoding ID or -1 to indicate that
 * the caller will check whether it accepts the ID.
 *
 * Note: the errors here are not really user-facing, so elog instead of
 * ereport seems sufficient.  Also, we trust that the "expected" encoding
 * arguments are valid encoding IDs, but we don't trust the actuals.
 */
void
check_encoding_conversion_args(int src_encoding,
                                             int dest_encoding,
                                             int len,
                                             int expected_src_encoding,
                                             int expected_dest_encoding)
{
      if (!PG_VALID_ENCODING(src_encoding))
            elog(ERROR, "invalid source encoding ID: %d", src_encoding);
      if (src_encoding != expected_src_encoding && expected_src_encoding >= 0)
            elog(ERROR, "expected source encoding \"%s\", but got \"%s\"",
                   pg_enc2name_tbl[expected_src_encoding].name,
                   pg_enc2name_tbl[src_encoding].name);
      if (!PG_VALID_ENCODING(dest_encoding))
            elog(ERROR, "invalid destination encoding ID: %d", dest_encoding);
      if (dest_encoding != expected_dest_encoding && expected_dest_encoding >= 0)
            elog(ERROR, "expected destination encoding \"%s\", but got \"%s\"",
                   pg_enc2name_tbl[expected_dest_encoding].name,
                   pg_enc2name_tbl[dest_encoding].name);
      if (len < 0)
            elog(ERROR, "encoding conversion length must not be negative");
}

/*
 * report_invalid_encoding: complain about invalid multibyte character
 *
 * note: len is remaining length of string, not length of character;
 * len must be greater than zero, as we always examine the first byte.
 */
void
report_invalid_encoding(int encoding, const char *mbstr, int len)
{
      int               l = pg_encoding_mblen(encoding, mbstr);
      char        buf[8 * 2 + 1];
      char     *p = buf;
      int               j,
                        jlimit;

      jlimit = Min(l, len);
      jlimit = Min(jlimit, 8);      /* prevent buffer overrun */

      for (j = 0; j < jlimit; j++)
            p += sprintf(p, "%02x", (unsigned char) mbstr[j]);

      ereport(ERROR,
                  (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
                   errmsg("invalid byte sequence for encoding \"%s\": 0x%s",
                              pg_enc2name_tbl[encoding].name,
                              buf),
              errhint("This error can also happen if the byte sequence does not "
                  "match the encoding expected by the server, which is controlled "
                          "by \"client_encoding\".")));
}

/*
 * report_untranslatable_char: complain about untranslatable character
 *
 * note: len is remaining length of string, not length of character;
 * len must be greater than zero, as we always examine the first byte.
 */
void
report_untranslatable_char(int src_encoding, int dest_encoding,
                                       const char *mbstr, int len)
{
      int               l = pg_encoding_mblen(src_encoding, mbstr);
      char        buf[8 * 2 + 1];
      char     *p = buf;
      int               j,
                        jlimit;

      jlimit = Min(l, len);
      jlimit = Min(jlimit, 8);      /* prevent buffer overrun */

      for (j = 0; j < jlimit; j++)
            p += sprintf(p, "%02x", (unsigned char) mbstr[j]);

      ereport(ERROR,
                  (errcode(ERRCODE_UNTRANSLATABLE_CHARACTER),
                   errmsg("character 0x%s of encoding \"%s\" has no equivalent in \"%s\"",
                              buf,
                              pg_enc2name_tbl[src_encoding].name,
                              pg_enc2name_tbl[dest_encoding].name)));
}

#endif

Generated by  Doxygen 1.6.0   Back to index