1 // utf8.h: utilities for converting to and from UTF-8
3 // Copyright (C) 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
5 // This program is free software; you can redistribute it and/or modify
6 // it under the terms of the GNU General Public License as published by
7 // the Free Software Foundation; either version 3 of the License, or
8 // (at your option) any later version.
10 // This program is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 // GNU General Public License for more details.
15 // You should have received a copy of the GNU General Public License
16 // along with this program; if not, write to the Free Software
17 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19 // Based on the public domain work of Thatcher Ulrich <tu@tulrich.com> 2004
25 #include <cstdint> // for C99 int types
28 #include "dsodefs.h" // For DSOEXPORT
30 // Android doesn't have any support for wide characters at all.
35 ,std::char_traits
<wchar_t>
36 ,std::allocator
<wchar_t> >
43 /// Utilities to convert between std::string and std::wstring.
45 /// Strings in Gnash are generally stored as std::strings.
46 /// We have to deal, however, with characters larger than standard
47 /// ASCII (128), which can be encoded in two different ways.
49 /// SWF6 and later use UTF-8, encoded as multibyte characters and
50 /// allowing many thousands of unique codes. Multibyte characters are
51 /// difficult to handle, as their length - used for many string
52 /// operations - is not certain without parsing the string.
53 /// Converting the string to a wstring (generally a uint32_t - the
54 /// pp seems only to handle characters up to 65535 - two bytes is
55 /// the minimum size of a wchar) facilitates string operations, as
56 /// the length of the string is equal to the number of valid characters.
58 /// SWF5 and earlier, however, used the ISO-8859 specification,
59 /// allowing the standard 128 ASCII characters plus 128 extra
60 /// characters that depend on the particular subset of ISO-8859.
61 /// Characters are 8 bits, not the ASCII standard 7. SWF5 cannot
62 /// handle multi-byte characters without special functions.
64 /// It is important that SWF5 can distinguish between the two encodings,
65 /// so we cannot convert all strings to UTF-8.
67 /// Please note that, although this is called utf8, what the Adobe
68 /// player uses is only loosely related to real unicode, so the
69 /// encoding support here is correspondingly non-standard.
72 /// Converts a std::string with multibyte characters into a std::wstring.
74 /// @return a version-dependent wstring.
75 /// @param str the canonical string to convert.
76 /// @param version the SWF version, used to decide how to decode the string.
78 /// For SWF5, UTF-8 (or any other) multibyte encoded characters are
79 /// converted char by char, mangling the string.
80 DSOEXPORT
std::wstring
decodeCanonicalString(const std::string
& str
, int version
);
82 /// Converts a std::wstring into canonical std::string.
84 /// @return a version-dependent encoded std::string.
85 /// @param wstr the wide string to convert.
86 /// @param version the SWF version, used to decide how to encode the string.
88 /// For SWF 5, each character is stored as an 8-bit (at least) char, rather
89 /// than converting it to a canonical UTF-8 byte sequence. Gnash can then
90 /// distinguish between 8-bit characters, which it handles correctly, and
91 /// multi-byte characters, which are regarded as multiple characters for
93 DSOEXPORT
std::string
encodeCanonicalString(const std::wstring
& wstr
, int version
);
95 /// Return the next Unicode character in the UTF-8 encoded string.
97 /// Invalid UTF-8 sequences produce a U+FFFD character
98 /// as output. Advances string iterator past the character
99 /// returned, unless the returned character is '\0', in which
100 /// case the iterator does not advance.
101 DSOEXPORT
std::uint32_t decodeNextUnicodeCharacter(std::string::const_iterator
& it
,
102 const std::string::const_iterator
& e
);
104 /// \brief Encodes the given wide character into a canonical
105 /// string, theoretically up to 6 chars in length.
106 DSOEXPORT
std::string
encodeUnicodeCharacter(std::uint32_t ucs_character
);
108 /// Encodes the given wide character into an at least 8-bit character.
110 /// Allows storage of Latin1 (ISO-8859-1) characters. This
111 /// is the format of SWF5 and below.
112 DSOEXPORT
std::string
encodeLatin1Character(std::uint32_t ucsCharacter
);
127 /// Interpret (and skip) Byte Order Mark in input stream
129 /// This function takes a pointer to a buffer and returns
130 /// the start of actual data after an eventual BOM.
131 /// No conversion is performed, no bytes copy, just skipping of
132 /// the BOM snippet and interpretation of it returned to the
133 /// encoding input parameter.
135 /// See http://en.wikipedia.org/wiki/Byte-order_mark
138 /// The input buffer.
141 /// Size of the input buffer, will be decremented by the
142 /// size of the BOM, if any.
145 /// Output parameter, will always be set.
146 /// encUNSPECIFIED if no BOM is found.
149 /// A pointer either equal to 'in' or some bytes inside it.
151 DSOEXPORT
const char* stripBOM(const char* in
, size_t& size
,
152 TextEncoding
& encoding
);
154 /// Return name of a text encoding
155 DSOEXPORT
const char* textEncodingName(TextEncoding enc
);
158 ENCGUESS_UNICODE
= 0,
163 /// Common code for guessing at the encoding of random text, between
164 // Shift-Jis, UTF8, and other. Puts the DisplayObject count in length,
165 // and the offsets to the DisplayObjects in offsets, if offsets is not NULL.
166 // If not NULL, offsets should be at least s.length().
167 // offsets are not accurate if the return value is GUESSENC_OTHER
169 /// TODO: It's doubtful if this even works, and it may not be useful at
171 DSOEXPORT EncodingGuess
guessEncoding(const std::string
& s
, int& length
,
172 std::vector
<int>& offsets
);
185 // indent-tabs-mode: t