librime: checked.h Source File

| librime 1.2 Rime Input Method Engine, the core library

checked.h
Go to the documentation of this file.
1// Copyright 2006-2016 Nemanja Trifunovic
2
3/*
4Permission is hereby granted, free of charge, to any person or organization
5obtaining a copy of the software and accompanying documentation covered by
6this license (the "Software") to use, reproduce, display, distribute,
7execute, and transmit the Software, and to prepare derivative works of the
8Software, and to permit third-parties to whom the Software is furnished to
9do so, all subject to the following:
10
11The copyright notices in the Software and this entire statement, including
12the above license grant, this restriction and the following disclaimer,
13must be included in all copies of the Software, in whole or in part, and
14all derivative works of the Software, unless such copies or derivative
15works are solely in the form of machine-executable object code generated by
16a source language processor.
17
18THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
21SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
22FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
23ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24DEALINGS IN THE SOFTWARE.
25*/
26
27
28#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
29#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
30
31#include "core.h"
32#include <stdexcept>
33
34 namespace utf8
35{
36 // Base for the exceptions that may be thrown from the library
37 class exception : public ::std::exception {
38 };
39
40 // Exceptions that may be thrown from the library functions.
41 class invalid_code_point : public exception {
42 uint32_t cp;
43 public:
44 invalid_code_point(uint32_t codepoint) : cp(codepoint) {}
45 virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid code point"; }
46 uint32_t code_point() const {return cp;}
47 };
48
49 class invalid_utf8 : public exception {
50 uint8_t u8;
51 public:
52 invalid_utf8 (uint8_t u) : u8(u) {}
53 invalid_utf8 (char c) : u8(static_cast<uint8_t>(c)) {}
54 virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-8"; }
55 uint8_t utf8_octet() const {return u8;}
56 };
57
58 class invalid_utf16 : public exception {
59 uint16_t u16;
60 public:
61 invalid_utf16 (uint16_t u) : u16(u) {}
62 virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-16"; }
63 uint16_t utf16_word() const {return u16;}
64 };
65
66 class not_enough_room : public exception {
67 public:
68 virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Not enough space"; }
69 };
70
72
73 template <typename octet_iterator>
74 octet_iterator append(uint32_t cp, octet_iterator result)
75 {
76 if (!utf8::internal::is_code_point_valid(cp))
77 throw invalid_code_point(cp);
78
79 return internal::append(cp, result);
80 }
81
82 template <typename octet_iterator, typename output_iterator>
83 output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
84 {
85 while (start != end) {
86 octet_iterator sequence_start = start;
87 internal::utf_error err_code = utf8::internal::validate_next(start, end);
88 switch (err_code) {
89 case internal::UTF8_OK :
90 for (octet_iterator it = sequence_start; it != start; ++it)
91 *out++ = *it;
92 break;
93 case internal::NOT_ENOUGH_ROOM:
94 out = utf8::append (replacement, out);
95 start = end;
96 break;
97 case internal::INVALID_LEAD:
98 out = utf8::append (replacement, out);
99 ++start;
100 break;
101 case internal::INCOMPLETE_SEQUENCE:
102 case internal::OVERLONG_SEQUENCE:
103 case internal::INVALID_CODE_POINT:
104 out = utf8::append (replacement, out);
105 ++start;
106 // just one replacement mark for the sequence
107 while (start != end && utf8::internal::is_trail(*start))
108 ++start;
109 break;
110 }
111 }
112 return out;
113 }
114
115 template <typename octet_iterator, typename output_iterator>
116 inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
117 {
118 static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
119 return utf8::replace_invalid(start, end, out, replacement_marker);
120 }
121
122 template <typename octet_iterator>
123 uint32_t next(octet_iterator& it, octet_iterator end)
124 {
125 uint32_t cp = 0;
126 internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
127 switch (err_code) {
128 case internal::UTF8_OK :
129 break;
130 case internal::NOT_ENOUGH_ROOM :
131 throw not_enough_room();
132 case internal::INVALID_LEAD :
133 case internal::INCOMPLETE_SEQUENCE :
134 case internal::OVERLONG_SEQUENCE :
135 throw invalid_utf8(static_cast<uint8_t>(*it));
136 case internal::INVALID_CODE_POINT :
137 throw invalid_code_point(cp);
138 }
139 return cp;
140 }
141
142 template <typename octet_iterator>
143 uint32_t peek_next(octet_iterator it, octet_iterator end)
144 {
145 return utf8::next(it, end);
146 }
147
148 template <typename octet_iterator>
149 uint32_t prior(octet_iterator& it, octet_iterator start)
150 {
151 // can't do much if it == start
152 if (it == start)
153 throw not_enough_room();
154
155 octet_iterator end = it;
156 // Go back until we hit either a lead octet or start
157 while (utf8::internal::is_trail(*(--it)))
158 if (it == start)
159 throw invalid_utf8(*it); // error - no lead byte in the sequence
160 return utf8::peek_next(it, end);
161 }
162
163 template <typename octet_iterator, typename distance_type>
164 void advance (octet_iterator& it, distance_type n, octet_iterator end)
165 {
166 const distance_type zero(0);
167 if (n < zero) {
168 // backward
169 for (distance_type i = n; i < zero; ++i)
170 utf8::prior(it, end);
171 } else {
172 // forward
173 for (distance_type i = zero; i < n; ++i)
174 utf8::next(it, end);
175 }
176 }
177
178 template <typename octet_iterator>
179 typename std::iterator_traits<octet_iterator>::difference_type
180 distance (octet_iterator first, octet_iterator last)
181 {
182 typename std::iterator_traits<octet_iterator>::difference_type dist;
183 for (dist = 0; first < last; ++dist)
184 utf8::next(first, last);
185 return dist;
186 }
187
188 template <typename u16bit_iterator, typename octet_iterator>
189 octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
190 {
191 while (start != end) {
192 uint32_t cp = utf8::internal::mask16(*start++);
193 // Take care of surrogate pairs first
194 if (utf8::internal::is_lead_surrogate(cp)) {
195 if (start != end) {
196 uint32_t trail_surrogate = utf8::internal::mask16(*start++);
197 if (utf8::internal::is_trail_surrogate(trail_surrogate))
198 cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
199 else
200 throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
201 }
202 else
203 throw invalid_utf16(static_cast<uint16_t>(cp));
204
205 }
206 // Lone trail surrogate
207 else if (utf8::internal::is_trail_surrogate(cp))
208 throw invalid_utf16(static_cast<uint16_t>(cp));
209
210 result = utf8::append(cp, result);
211 }
212 return result;
213 }
214
215 template <typename u16bit_iterator, typename octet_iterator>
216 u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
217 {
218 while (start < end) {
219 uint32_t cp = utf8::next(start, end);
220 if (cp > 0xffff) { //make a surrogate pair
221 *result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
222 *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
223 }
224 else
225 *result++ = static_cast<uint16_t>(cp);
226 }
227 return result;
228 }
229
230 template <typename octet_iterator, typename u32bit_iterator>
231 octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
232 {
233 while (start != end)
234 result = utf8::append(*(start++), result);
235
236 return result;
237 }
238
239 template <typename octet_iterator, typename u32bit_iterator>
240 u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
241 {
242 while (start < end)
243 (*result++) = utf8::next(start, end);
244
245 return result;
246 }
247
248 // The iterator class
249 template <typename octet_iterator>
250 class iterator {
251 octet_iterator it;
252 octet_iterator range_start;
253 octet_iterator range_end;
254 public:
255 typedef uint32_t value_type;
256 typedef uint32_t* pointer;
257 typedef uint32_t& reference;
258 typedef std::ptrdiff_t difference_type;
259 typedef std::bidirectional_iterator_tag iterator_category;
260 iterator () {}
261 explicit iterator (const octet_iterator& octet_it,
262 const octet_iterator& rangestart,
263 const octet_iterator& rangeend) :
264 it(octet_it), range_start(rangestart), range_end(rangeend)
265 {
266 if (it < range_start || it > range_end)
267 throw std::out_of_range("Invalid utf-8 iterator position");
268 }
269 // the default "big three" are OK
270 octet_iterator base () const { return it; }
271 uint32_t operator * () const
272 {
273 octet_iterator temp = it;
274 return utf8::next(temp, range_end);
275 }
276 bool operator == (const iterator& rhs) const
277 {
278 if (range_start != rhs.range_start || range_end != rhs.range_end)
279 throw std::logic_error("Comparing utf-8 iterators defined with different ranges");
280 return (it == rhs.it);
281 }
282 bool operator != (const iterator& rhs) const
283 {
284 return !(operator == (rhs));
285 }
286 iterator& operator ++ ()
287 {
288 utf8::next(it, range_end);
289 return *this;
290 }
291 iterator operator ++ (int)
292 {
293 iterator temp = *this;
294 utf8::next(it, range_end);
295 return temp;
296 }
297 iterator& operator -- ()
298 {
299 utf8::prior(it, range_start);
300 return *this;
301 }
302 iterator operator -- (int)
303 {
304 iterator temp = *this;
305 utf8::prior(it, range_start);
306 return temp;
307 }
308 }; // class iterator
309
310} // namespace utf8
311
312#if UTF_CPP_CPLUSPLUS >= 201703L // C++ 17 or later
313#include "cpp17.h"
314#elif UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later
315#include "cpp11.h"
316#endif // C++ 11 or later
317
318#endif //header guard
319
core.h
UTF_CPP_OVERRIDE
#define UTF_CPP_OVERRIDE
Definition core.h:45
UTF_CPP_NOEXCEPT
#define UTF_CPP_NOEXCEPT
Definition core.h:46
cpp17.h
utf8
Definition checked.h:35
utf8::replace_invalid
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
Definition checked.h:83
utf8::append
octet_iterator append(uint32_t cp, octet_iterator result)
The library API - functions intended to be called by the users.
Definition checked.h:74
utf8::advance
void advance(octet_iterator &it, distance_type n, octet_iterator end)
Definition checked.h:164
utf8::peek_next
uint32_t peek_next(octet_iterator it, octet_iterator end)
Definition checked.h:143
utf8::utf32to8
octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end, octet_iterator result)
Definition checked.h:231
utf8::utf8to16
u16bit_iterator utf8to16(octet_iterator start, octet_iterator end, u16bit_iterator result)
Definition checked.h:216
utf8::utf8to32
u32bit_iterator utf8to32(octet_iterator start, octet_iterator end, u32bit_iterator result)
Definition checked.h:240
utf8::uint32_t
unsigned int uint32_t
Definition core.h:57
utf8::prior
uint32_t prior(octet_iterator &it, octet_iterator start)
Definition checked.h:149
utf8::uint8_t
unsigned char uint8_t
Definition core.h:55
utf8::uint16_t
unsigned short uint16_t
Definition core.h:56
utf8::distance
std::iterator_traits< octet_iterator >::difference_type distance(octet_iterator first, octet_iterator last)
Definition checked.h:180
utf8::utf16to8
octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end, octet_iterator result)
Definition checked.h:189
utf8::next
uint32_t next(octet_iterator &it, octet_iterator end)
Definition checked.h:123
utf8::internal::is_lead_surrogate
bool is_lead_surrogate(u16 cp)
Definition core.h:92
utf8::internal::is_trail_surrogate
bool is_trail_surrogate(u16 cp)
Definition core.h:98
utf8::internal::SURROGATE_OFFSET
const uint32_t SURROGATE_OFFSET
Definition core.h:70
utf8::internal::LEAD_OFFSET
const uint16_t LEAD_OFFSET
Definition core.h:69
utf8::internal::TRAIL_SURROGATE_MIN
const uint16_t TRAIL_SURROGATE_MIN
Definition core.h:67
utf8::internal::is_code_point_valid
bool is_code_point_valid(u32 cp)
Definition core.h:110
utf8::internal::utf_error
utf_error
Definition core.h:151
utf8::internal::INCOMPLETE_SEQUENCE
@ INCOMPLETE_SEQUENCE
Definition core.h:151
utf8::internal::INVALID_LEAD
@ INVALID_LEAD
Definition core.h:151
utf8::internal::OVERLONG_SEQUENCE
@ OVERLONG_SEQUENCE
Definition core.h:151
utf8::internal::INVALID_CODE_POINT
@ INVALID_CODE_POINT
Definition core.h:151
utf8::internal::NOT_ENOUGH_ROOM
@ NOT_ENOUGH_ROOM
Definition core.h:151
utf8::internal::UTF8_OK
@ UTF8_OK
Definition core.h:151
utf8::internal::mask16
uint16_t mask16(u16_type oc)
Definition core.h:81
utf8::internal::is_trail
bool is_trail(octet_type oc)
Definition core.h:86
utf8::internal::validate_next
utf_error validate_next(octet_iterator &it, octet_iterator end, uint32_t &code_point)
Definition core.h:240
utf8::internal::append
octet_iterator append(uint32_t cp, octet_iterator result)
Definition core.h:304
utf8::exception
Definition checked.h:37
utf8::invalid_code_point::code_point
uint32_t code_point() const
Definition checked.h:46
utf8::invalid_code_point::what
virtual const char * what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE
Definition checked.h:45
utf8::invalid_code_point::invalid_code_point
invalid_code_point(uint32_t codepoint)
Definition checked.h:44
utf8::invalid_utf8::invalid_utf8
invalid_utf8(uint8_t u)
Definition checked.h:52
utf8::invalid_utf8::invalid_utf8
invalid_utf8(char c)
Definition checked.h:53
utf8::invalid_utf8::what
virtual const char * what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE
Definition checked.h:54
utf8::invalid_utf8::utf8_octet
uint8_t utf8_octet() const
Definition checked.h:55
utf8::invalid_utf16::utf16_word
uint16_t utf16_word() const
Definition checked.h:63
utf8::invalid_utf16::what
virtual const char * what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE
Definition checked.h:62
utf8::invalid_utf16::invalid_utf16
invalid_utf16(uint16_t u)
Definition checked.h:61
utf8::not_enough_room
Definition checked.h:66
utf8::not_enough_room::what
virtual const char * what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE
Definition checked.h:68
utf8::iterator::operator*
uint32_t operator*() const
Definition checked.h:271
utf8::iterator::reference
uint32_t & reference
Definition checked.h:257
utf8::iterator::operator==
bool operator==(const iterator &rhs) const
Definition checked.h:276
utf8::iterator::base
octet_iterator base() const
Definition checked.h:270
utf8::iterator::difference_type
std::ptrdiff_t difference_type
Definition checked.h:258
utf8::iterator::iterator
iterator()
Definition checked.h:260
utf8::iterator::operator--
iterator & operator--()
Definition checked.h:297
utf8::iterator::value_type
uint32_t value_type
Definition checked.h:255
utf8::iterator::iterator_category
std::bidirectional_iterator_tag iterator_category
Definition checked.h:259
utf8::iterator::operator++
iterator & operator++()
Definition checked.h:286
utf8::iterator::pointer
uint32_t * pointer
Definition checked.h:256
utf8::iterator::operator!=
bool operator!=(const iterator &rhs) const
Definition checked.h:282
utf8::iterator::iterator
iterator(const octet_iterator &octet_it, const octet_iterator &rangestart, const octet_iterator &rangeend)
Definition checked.h:261