librime: core.h Source File

| librime 1.2 Rime Input Method Engine, the core library

core.h
Go to the documentation of this file.
1// Copyright 2006 Nemanja Trifunovic
2
3/*
4Permission is hereby granted, free of charge, to any person or organization
5obtaining a copy of the software and accompanying documentation covered by
6this license (the "Software") to use, reproduce, display, distribute,
7execute, and transmit the Software, and to prepare derivative works of the
8Software, and to permit third-parties to whom the Software is furnished to
9do so, all subject to the following:
10
11The copyright notices in the Software and this entire statement, including
12the above license grant, this restriction and the following disclaimer,
13must be included in all copies of the Software, in whole or in part, and
14all derivative works of the Software, unless such copies or derivative
15works are solely in the form of machine-executable object code generated by
16a source language processor.
17
18THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
21SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
22FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
23ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24DEALINGS IN THE SOFTWARE.
25*/
26
27
28#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
29#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
30
31#include <iterator>
32
33// Determine the C++ standard version.
34// If the user defines UTF_CPP_CPLUSPLUS, use that.
35// Otherwise, trust the unreliable predefined macro __cplusplus
36
37#if !defined UTF_CPP_CPLUSPLUS
38 #define UTF_CPP_CPLUSPLUS __cplusplus
39#endif
40
41#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later
42 #define UTF_CPP_OVERRIDE override
43 #define UTF_CPP_NOEXCEPT noexcept
44#else // C++ 98/03
45 #define UTF_CPP_OVERRIDE
46 #define UTF_CPP_NOEXCEPT throw()
47#endif // C++ 11 or later
48
49
50namespace utf8
51{
52 // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
53 // You may need to change them to match your system.
54 // These typedefs have the same names as ones from cstdint, or boost/cstdint
55 typedef unsigned char uint8_t;
56 typedef unsigned short uint16_t;
57 typedef unsigned int uint32_t;
58
59// Helper code - not intended to be directly called by the library users. May be changed at any time
60 namespace internal
61{
62 // Unicode constants
63 // Leading (high) surrogates: 0xd800 - 0xdbff
64 // Trailing (low) surrogates: 0xdc00 - 0xdfff
65 const uint16_t LEAD_SURROGATE_MIN = 0xd800u;
66 const uint16_t LEAD_SURROGATE_MAX = 0xdbffu;
67 const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
68 const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
69 const uint16_t LEAD_OFFSET = 0xd7c0u; // LEAD_SURROGATE_MIN - (0x10000 >> 10)
70 const uint32_t SURROGATE_OFFSET = 0xfca02400u; // 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN
71
72 // Maximum valid value for a Unicode code point
73 const uint32_t CODE_POINT_MAX = 0x0010ffffu;
74
75 template<typename octet_type>
76 inline uint8_t mask8(octet_type oc)
77 {
78 return static_cast<uint8_t>(0xff & oc);
79 }
80 template<typename u16_type>
81 inline uint16_t mask16(u16_type oc)
82 {
83 return static_cast<uint16_t>(0xffff & oc);
84 }
85 template<typename octet_type>
86 inline bool is_trail(octet_type oc)
87 {
88 return ((utf8::internal::mask8(oc) >> 6) == 0x2);
89 }
90
91 template <typename u16>
92 inline bool is_lead_surrogate(u16 cp)
93 {
94 return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
95 }
96
97 template <typename u16>
98 inline bool is_trail_surrogate(u16 cp)
99 {
100 return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
101 }
102
103 template <typename u16>
104 inline bool is_surrogate(u16 cp)
105 {
106 return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
107 }
108
109 template <typename u32>
110 inline bool is_code_point_valid(u32 cp)
111 {
112 return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
113 }
114
115 template <typename octet_iterator>
116 inline typename std::iterator_traits<octet_iterator>::difference_type
117 sequence_length(octet_iterator lead_it)
118 {
119 uint8_t lead = utf8::internal::mask8(*lead_it);
120 if (lead < 0x80)
121 return 1;
122 else if ((lead >> 5) == 0x6)
123 return 2;
124 else if ((lead >> 4) == 0xe)
125 return 3;
126 else if ((lead >> 3) == 0x1e)
127 return 4;
128 else
129 return 0;
130 }
131
132 template <typename octet_difference_type>
133 inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
134 {
135 if (cp < 0x80) {
136 if (length != 1)
137 return true;
138 }
139 else if (cp < 0x800) {
140 if (length != 2)
141 return true;
142 }
143 else if (cp < 0x10000) {
144 if (length != 3)
145 return true;
146 }
147
148 return false;
149 }
150
151 enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};
152
154 template <typename octet_iterator>
155 utf_error increase_safely(octet_iterator& it, octet_iterator end)
156 {
157 if (++it == end)
158 return NOT_ENOUGH_ROOM;
159
160 if (!utf8::internal::is_trail(*it))
161 return INCOMPLETE_SEQUENCE;
162
163 return UTF8_OK;
164 }
165
166 #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
167
169 template <typename octet_iterator>
170 utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
171 {
172 if (it == end)
173 return NOT_ENOUGH_ROOM;
174
175 code_point = utf8::internal::mask8(*it);
176
177 return UTF8_OK;
178 }
179
180 template <typename octet_iterator>
181 utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
182 {
183 if (it == end)
184 return NOT_ENOUGH_ROOM;
185
186 code_point = utf8::internal::mask8(*it);
187
188 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
189
190 code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);
191
192 return UTF8_OK;
193 }
194
195 template <typename octet_iterator>
196 utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
197 {
198 if (it == end)
199 return NOT_ENOUGH_ROOM;
200
201 code_point = utf8::internal::mask8(*it);
202
203 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
204
205 code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
206
207 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
208
209 code_point += (*it) & 0x3f;
210
211 return UTF8_OK;
212 }
213
214 template <typename octet_iterator>
215 utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
216 {
217 if (it == end)
218 return NOT_ENOUGH_ROOM;
219
220 code_point = utf8::internal::mask8(*it);
221
222 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
223
224 code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
225
226 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
227
228 code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
229
230 UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
231
232 code_point += (*it) & 0x3f;
233
234 return UTF8_OK;
235 }
236
237 #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
238
239 template <typename octet_iterator>
240 utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
241 {
242 if (it == end)
243 return NOT_ENOUGH_ROOM;
244
245 // Save the original value of it so we can go back in case of failure
246 // Of course, it does not make much sense with i.e. stream iterators
247 octet_iterator original_it = it;
248
249 uint32_t cp = 0;
250 // Determine the sequence length based on the lead octet
251 typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
252 const octet_difference_type length = utf8::internal::sequence_length(it);
253
254 // Get trail octets and calculate the code point
255 utf_error err = UTF8_OK;
256 switch (length) {
257 case 0:
258 return INVALID_LEAD;
259 case 1:
260 err = utf8::internal::get_sequence_1(it, end, cp);
261 break;
262 case 2:
263 err = utf8::internal::get_sequence_2(it, end, cp);
264 break;
265 case 3:
266 err = utf8::internal::get_sequence_3(it, end, cp);
267 break;
268 case 4:
269 err = utf8::internal::get_sequence_4(it, end, cp);
270 break;
271 }
272
273 if (err == UTF8_OK) {
274 // Decoding succeeded. Now, security checks...
275 if (utf8::internal::is_code_point_valid(cp)) {
276 if (!utf8::internal::is_overlong_sequence(cp, length)){
277 // Passed! Return here.
278 code_point = cp;
279 ++it;
280 return UTF8_OK;
281 }
282 else
283 err = OVERLONG_SEQUENCE;
284 }
285 else
286 err = INVALID_CODE_POINT;
287 }
288
289 // Failure branch - restore the original value of the iterator
290 it = original_it;
291 return err;
292 }
293
294 template <typename octet_iterator>
295 inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
296 uint32_t ignored;
297 return utf8::internal::validate_next(it, end, ignored);
298 }
299
300 // Internal implementation of both checked and unchecked append() function
301 // This function will be invoked by the overloads below, as they will know
302 // the octet_type.
303 template <typename octet_iterator, typename octet_type>
304 octet_iterator append(uint32_t cp, octet_iterator result) {
305 if (cp < 0x80) // one octet
306 *(result++) = static_cast<octet_type>(cp);
307 else if (cp < 0x800) { // two octets
308 *(result++) = static_cast<octet_type>((cp >> 6) | 0xc0);
309 *(result++) = static_cast<octet_type>((cp & 0x3f) | 0x80);
310 }
311 else if (cp < 0x10000) { // three octets
312 *(result++) = static_cast<octet_type>((cp >> 12) | 0xe0);
313 *(result++) = static_cast<octet_type>(((cp >> 6) & 0x3f) | 0x80);
314 *(result++) = static_cast<octet_type>((cp & 0x3f) | 0x80);
315 }
316 else { // four octets
317 *(result++) = static_cast<octet_type>((cp >> 18) | 0xf0);
318 *(result++) = static_cast<octet_type>(((cp >> 12) & 0x3f)| 0x80);
319 *(result++) = static_cast<octet_type>(((cp >> 6) & 0x3f) | 0x80);
320 *(result++) = static_cast<octet_type>((cp & 0x3f) | 0x80);
321 }
322 return result;
323 }
324
325 // One of the following overloads will be invoked from the API calls
326
327 // A simple (but dangerous) case: the caller appends byte(s) to a char array
328 inline char* append(uint32_t cp, char* result) {
329 return append<char*, char>(cp, result);
330 }
331
332 // Hopefully, most common case: the caller uses back_inserter
333 // i.e. append(cp, std::back_inserter(str));
334 template<typename container_type>
335 std::back_insert_iterator<container_type> append
336 (uint32_t cp, std::back_insert_iterator<container_type> result) {
337 return append<std::back_insert_iterator<container_type>,
338 typename container_type::value_type>(cp, result);
339 }
340
341 // The caller uses some other kind of output operator - not covered above
342 // Note that in this case we are not able to determine octet_type
343 // so we assume it's uint_8; that can cause a conversion warning if we are wrong.
344 template <typename octet_iterator>
345 octet_iterator append(uint32_t cp, octet_iterator result) {
346 return append<octet_iterator, uint8_t>(cp, result);
347 }
348
349} // namespace internal
350
352
353 // Byte order mark
354 const uint8_t bom[] = {0xef, 0xbb, 0xbf};
355
356 template <typename octet_iterator>
357 octet_iterator find_invalid(octet_iterator start, octet_iterator end)
358 {
359 octet_iterator result = start;
360 while (result != end) {
361 utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end);
362 if (err_code != internal::UTF8_OK)
363 return result;
364 }
365 return result;
366 }
367
368 template <typename octet_iterator>
369 inline bool is_valid(octet_iterator start, octet_iterator end)
370 {
371 return (utf8::find_invalid(start, end) == end);
372 }
373
374 template <typename octet_iterator>
375 inline bool starts_with_bom (octet_iterator it, octet_iterator end)
376 {
377 return (
378 ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) &&
379 ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
380 ((it != end) && (utf8::internal::mask8(*it)) == bom[2])
381 );
382 }
383} // namespace utf8
384
385#endif // header guard
386
387
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END)
Definition core.h:166
utf8
Definition checked.h:35
utf8::starts_with_bom
bool starts_with_bom(octet_iterator it, octet_iterator end)
Definition core.h:375
utf8::uint32_t
unsigned int uint32_t
Definition core.h:57
utf8::uint8_t
unsigned char uint8_t
Definition core.h:55
utf8::uint16_t
unsigned short uint16_t
Definition core.h:56
utf8::bom
const uint8_t bom[]
The library API - functions intended to be called by the users.
Definition core.h:354
utf8::is_valid
bool is_valid(octet_iterator start, octet_iterator end)
Definition core.h:369
utf8::find_invalid
octet_iterator find_invalid(octet_iterator start, octet_iterator end)
Definition core.h:357
utf8::internal
Definition core.h:61
utf8::internal::CODE_POINT_MAX
const uint32_t CODE_POINT_MAX
Definition core.h:73
utf8::internal::is_lead_surrogate
bool is_lead_surrogate(u16 cp)
Definition core.h:92
utf8::internal::is_trail_surrogate
bool is_trail_surrogate(u16 cp)
Definition core.h:98
utf8::internal::is_overlong_sequence
bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
Definition core.h:133
utf8::internal::is_surrogate
bool is_surrogate(u16 cp)
Definition core.h:104
utf8::internal::SURROGATE_OFFSET
const uint32_t SURROGATE_OFFSET
Definition core.h:70
utf8::internal::increase_safely
utf_error increase_safely(octet_iterator &it, octet_iterator end)
Helper for get_sequence_x.
Definition core.h:155
utf8::internal::TRAIL_SURROGATE_MAX
const uint16_t TRAIL_SURROGATE_MAX
Definition core.h:68
utf8::internal::LEAD_OFFSET
const uint16_t LEAD_OFFSET
Definition core.h:69
utf8::internal::get_sequence_1
utf_error get_sequence_1(octet_iterator &it, octet_iterator end, uint32_t &code_point)
get_sequence_x functions decode utf-8 sequences of the length x
Definition core.h:170
utf8::internal::TRAIL_SURROGATE_MIN
const uint16_t TRAIL_SURROGATE_MIN
Definition core.h:67
utf8::internal::is_code_point_valid
bool is_code_point_valid(u32 cp)
Definition core.h:110
utf8::internal::get_sequence_2
utf_error get_sequence_2(octet_iterator &it, octet_iterator end, uint32_t &code_point)
Definition core.h:181
utf8::internal::utf_error
utf_error
Definition core.h:151
utf8::internal::INCOMPLETE_SEQUENCE
@ INCOMPLETE_SEQUENCE
Definition core.h:151
utf8::internal::INVALID_LEAD
@ INVALID_LEAD
Definition core.h:151
utf8::internal::OVERLONG_SEQUENCE
@ OVERLONG_SEQUENCE
Definition core.h:151
utf8::internal::INVALID_CODE_POINT
@ INVALID_CODE_POINT
Definition core.h:151
utf8::internal::NOT_ENOUGH_ROOM
@ NOT_ENOUGH_ROOM
Definition core.h:151
utf8::internal::UTF8_OK
@ UTF8_OK
Definition core.h:151
utf8::internal::mask16
uint16_t mask16(u16_type oc)
Definition core.h:81
utf8::internal::is_trail
bool is_trail(octet_type oc)
Definition core.h:86
utf8::internal::validate_next
utf_error validate_next(octet_iterator &it, octet_iterator end, uint32_t &code_point)
Definition core.h:240
utf8::internal::get_sequence_3
utf_error get_sequence_3(octet_iterator &it, octet_iterator end, uint32_t &code_point)
Definition core.h:196
utf8::internal::LEAD_SURROGATE_MIN
const uint16_t LEAD_SURROGATE_MIN
Definition core.h:65
utf8::internal::get_sequence_4
utf_error get_sequence_4(octet_iterator &it, octet_iterator end, uint32_t &code_point)
Definition core.h:215
utf8::internal::sequence_length
std::iterator_traits< octet_iterator >::difference_type sequence_length(octet_iterator lead_it)
Definition core.h:117
utf8::internal::mask8
uint8_t mask8(octet_type oc)
Definition core.h:76
utf8::internal::LEAD_SURROGATE_MAX
const uint16_t LEAD_SURROGATE_MAX
Definition core.h:66
utf8::internal::append
octet_iterator append(uint32_t cp, octet_iterator result)
Definition core.h:304