ext/scintilla/src/UniConversion.cxx

   1 // Scintilla source code edit control
   2 /** @file UniConversion.cxx
   3  ** Functions to handle UTF-8 and UTF-16 strings.
   4  **/
   5 // Copyright 1998-2001 by Neil Hodgson <neilh@scintilla.org>
   6 // The License.txt file describes the conditions under which this software may be distributed.
   7
   8 #include <stdlib.h>
   9
  10 #include "UniConversion.h"
  11
  12 #ifdef SCI_NAMESPACE
  13 using namespace Scintilla;
  14 #endif
  15
  16 #ifdef SCI_NAMESPACE
  17 namespace Scintilla {
  18 #endif
  19
  20 enum { SURROGATE_LEAD_FIRST = 0xD800 };
  21 enum { SURROGATE_TRAIL_FIRST = 0xDC00 };
  22 enum { SURROGATE_TRAIL_LAST = 0xDFFF };
  23
  24 unsigned int UTF8Length(const wchar_t *uptr, unsigned int tlen) {
  25         unsigned int len = 0;
  26         for (unsigned int i = 0; i < tlen && uptr[i];) {
  27                 unsigned int uch = uptr[i];
  28                 if (uch < 0x80) {
  29                         len++;
  30                 } else if (uch < 0x800) {
  31                         len += 2;
  32                 } else if ((uch >= SURROGATE_LEAD_FIRST) &&
  33                         (uch <= SURROGATE_TRAIL_LAST)) {
  34                         len += 4;
  35                         i++;
  36                 } else {
  37                         len += 3;
  38                 }
  39                 i++;
  40         }
  41         return len;
  42 }
  43
  44 void UTF8FromUTF16(const wchar_t *uptr, unsigned int tlen, char *putf, unsigned int len) {
  45         int k = 0;
  46         for (unsigned int i = 0; i < tlen && uptr[i];) {
  47                 unsigned int uch = uptr[i];
  48                 if (uch < 0x80) {
  49                         putf[k++] = static_cast<char>(uch);
  50                 } else if (uch < 0x800) {
  51                         putf[k++] = static_cast<char>(0xC0 | (uch >> 6));
  52                         putf[k++] = static_cast<char>(0x80 | (uch & 0x3f));
  53                 } else if ((uch >= SURROGATE_LEAD_FIRST) &&
  54                         (uch <= SURROGATE_TRAIL_LAST)) {
  55                         // Half a surrogate pair
  56                         i++;
  57                         unsigned int xch = 0x10000 + ((uch & 0x3ff) << 10) + (uptr[i] & 0x3ff);
  58                         putf[k++] = static_cast<char>(0xF0 | (xch >> 18));
  59                         putf[k++] = static_cast<char>(0x80 | ((xch >> 12) & 0x3f));
  60                         putf[k++] = static_cast<char>(0x80 | ((xch >> 6) & 0x3f));
  61                         putf[k++] = static_cast<char>(0x80 | (xch & 0x3f));
  62                 } else {
  63                         putf[k++] = static_cast<char>(0xE0 | (uch >> 12));
  64                         putf[k++] = static_cast<char>(0x80 | ((uch >> 6) & 0x3f));
  65                         putf[k++] = static_cast<char>(0x80 | (uch & 0x3f));
  66                 }
  67                 i++;
  68         }
  69         putf[len] = '\0';
  70 }
  71
  72 unsigned int UTF8CharLength(unsigned char ch) {
  73         if (ch < 0x80) {
  74                 return 1;
  75         } else if (ch < 0x80 + 0x40 + 0x20) {
  76                 return 2;
  77         } else if (ch < 0x80 + 0x40 + 0x20 + 0x10) {
  78                 return 3;
  79         } else {
  80                 return 4;
  81         }
  82 }
  83
  84 unsigned int UTF16Length(const char *s, unsigned int len) {
  85         unsigned int ulen = 0;
  86         unsigned int charLen;
  87         for (unsigned int i=0; i<len;) {
  88                 unsigned char ch = static_cast<unsigned char>(s[i]);
  89                 if (ch < 0x80) {
  90                         charLen = 1;
  91                 } else if (ch < 0x80 + 0x40 + 0x20) {
  92                         charLen = 2;
  93                 } else if (ch < 0x80 + 0x40 + 0x20 + 0x10) {
  94                         charLen = 3;
  95                 } else {
  96                         charLen = 4;
  97                         ulen++;
  98                 }
  99                 i += charLen;
 100                 ulen++;
 101         }
 102         return ulen;
 103 }
 104
 105 unsigned int UTF16FromUTF8(const char *s, unsigned int len, wchar_t *tbuf, unsigned int tlen) {
 106         unsigned int ui=0;
 107         const unsigned char *us = reinterpret_cast<const unsigned char *>(s);
 108         unsigned int i=0;
 109         while ((i<len) && (ui<tlen)) {
 110                 unsigned char ch = us[i++];
 111                 if (ch < 0x80) {
 112                         tbuf[ui] = ch;
 113                 } else if (ch < 0x80 + 0x40 + 0x20) {
 114                         tbuf[ui] = static_cast<wchar_t>((ch & 0x1F) << 6);
 115                         ch = us[i++];
 116                         tbuf[ui] = static_cast<wchar_t>(tbuf[ui] + (ch & 0x7F));
 117                 } else if (ch < 0x80 + 0x40 + 0x20 + 0x10) {
 118                         tbuf[ui] = static_cast<wchar_t>((ch & 0xF) << 12);
 119                         ch = us[i++];
 120                         tbuf[ui] = static_cast<wchar_t>(tbuf[ui] + ((ch & 0x7F) << 6));
 121                         ch = us[i++];
 122                         tbuf[ui] = static_cast<wchar_t>(tbuf[ui] + (ch & 0x7F));
 123                 } else {
 124                         // Outside the BMP so need two surrogates
 125                         int val = (ch & 0x7) << 18;
 126                         ch = us[i++];
 127                         val += (ch & 0x3F) << 12;
 128                         ch = us[i++];
 129                         val += (ch & 0x3F) << 6;
 130                         ch = us[i++];
 131                         val += (ch & 0x3F);
 132                         tbuf[ui] = static_cast<wchar_t>(((val - 0x10000) >> 10) + SURROGATE_LEAD_FIRST);
 133                         ui++;
 134                         tbuf[ui] = static_cast<wchar_t>((val & 0x3ff) + SURROGATE_TRAIL_FIRST);
 135                 }
 136                 ui++;
 137         }
 138         return ui;
 139 }
 140
 141 int UTF8BytesOfLead[256];
 142 static bool initialisedBytesOfLead = false;
 143
 144 static int BytesFromLead(int leadByte) {
 145         if (leadByte < 0xC2) {
 146                 // Single byte or invalid
 147                 return 1;
 148         } else if (leadByte < 0xE0) {
 149                 return 2;
 150         } else if (leadByte < 0xF0) {
 151                 return 3;
 152         } else if (leadByte < 0xF5) {
 153                 return 4;
 154         } else {
 155                 // Characters longer than 4 bytes not possible in current UTF-8
 156                 return 1;
 157         }
 158 }
 159
 160 void UTF8BytesOfLeadInitialise() {
 161         if (!initialisedBytesOfLead) {
 162                 for (int i=0; i<256; i++) {
 163                         UTF8BytesOfLead[i] = BytesFromLead(i);
 164                 }
 165                 initialisedBytesOfLead = true;
 166         }
 167 }
 168
 169 // Return both the width of the first character in the string and a status
 170 // saying whether it is valid or invalid.
 171 // Most invalid sequences return a width of 1 so are treated as isolated bytes but
 172 // the non-characters *FFFE, *FFFF and FDD0 .. FDEF return 3 or 4 as they can be
 173 // reasonably treated as code points in some circumstances. They will, however,
 174 // not have associated glyphs.
 175 int UTF8Classify(const unsigned char *us, int len) {
 176         // For the rules: http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8
 177         if (*us < 0x80) {
 178                 // Single bytes easy
 179                 return 1;
 180         } else if (*us > 0xf4) {
 181                 // Characters longer than 4 bytes not possible in current UTF-8
 182                 return UTF8MaskInvalid | 1;
 183         } else if (*us >= 0xf0) {
 184                 // 4 bytes
 185                 if (len < 4)
 186                         return UTF8MaskInvalid | 1;
 187                 if (UTF8IsTrailByte(us[1]) && UTF8IsTrailByte(us[2]) && UTF8IsTrailByte(us[3])) {
 188                         if (((us[1] & 0xf) == 0xf) && (us[2] == 0xbf) && ((us[3] == 0xbe) || (us[3] == 0xbf))) {
 189                                 // *FFFE or *FFFF non-character
 190                                 return UTF8MaskInvalid | 4;
 191                         }
 192                         if (*us == 0xf4) {
 193                                 // Check if encoding a value beyond the last Unicode character 10FFFF
 194                                 if (us[1] > 0x8f) {
 195                                         return UTF8MaskInvalid | 1;
 196                                 } else if (us[1] == 0x8f) {
 197                                         if (us[2] > 0xbf) {
 198                                                 return UTF8MaskInvalid | 1;
 199                                         } else if (us[2] == 0xbf) {
 200                                                 if (us[3] > 0xbf) {
 201                                                         return UTF8MaskInvalid | 1;
 202                                                 }
 203                                         }
 204                                 }
 205                         } else if ((*us == 0xf0) && ((us[1] & 0xf0) == 0x80)) {
 206                                 // Overlong
 207                                 return UTF8MaskInvalid | 1;
 208                         }
 209                         return 4;
 210                 } else {
 211                         return UTF8MaskInvalid | 1;
 212                 }
 213         } else if (*us >= 0xe0) {
 214                 // 3 bytes
 215                 if (len < 3)
 216                         return UTF8MaskInvalid | 1;
 217                 if (UTF8IsTrailByte(us[1]) && UTF8IsTrailByte(us[2])) {
 218                         if ((*us == 0xe0) && ((us[1] & 0xe0) == 0x80)) {
 219                                 // Overlong
 220                                 return UTF8MaskInvalid | 1;
 221                         }
 222                         if ((*us == 0xed) && ((us[1] & 0xe0) == 0xa0)) {
 223                                 // Surrogate
 224                                 return UTF8MaskInvalid | 1;
 225                         }
 226                         if ((*us == 0xef) && (us[1] == 0xbf) && (us[2] == 0xbe)) {
 227                                 // U+FFFE non-character - 3 bytes long
 228                                 return UTF8MaskInvalid | 3;
 229                         }
 230                         if ((*us == 0xef) && (us[1] == 0xbf) && (us[2] == 0xbf)) {
 231                                 // U+FFFF non-character - 3 bytes long
 232                                 return UTF8MaskInvalid | 3;
 233                         }
 234                         if ((*us == 0xef) && (us[1] == 0xb7) && (((us[2] & 0xf0) == 0x90) || ((us[2] & 0xf0) == 0xa0))) {
 235                                 // U+FDD0 .. U+FDEF
 236                                 return UTF8MaskInvalid | 3;
 237                         }
 238                         return 3;
 239                 } else {
 240                         return UTF8MaskInvalid | 1;
 241                 }
 242         } else if (*us >= 0xc2) {
 243                 // 2 bytes
 244                 if (len < 2)
 245                         return UTF8MaskInvalid | 1;
 246                 if (UTF8IsTrailByte(us[1])) {
 247                         return 2;
 248                 } else {
 249                         return UTF8MaskInvalid | 1;
 250                 }
 251         } else {
 252                 // 0xc0 .. 0xc1 is overlong encoding
 253                 // 0x80 .. 0xbf is trail byte
 254                 return UTF8MaskInvalid | 1;
 255         }
 256 }
 257
 258 int UTF8DrawBytes(const unsigned char *us, int len) {
 259         int utf8StatusNext = UTF8Classify(us, len);
 260         return (utf8StatusNext & UTF8MaskInvalid) ? 1 : (utf8StatusNext & UTF8MaskWidth);
 261 }
 262
 263 #ifdef SCI_NAMESPACE
 264 }
 265 #endif