6 * Use, modification and distribution are subject to the
7 * Boost Software License, Version 1.0. (See accompanying file
8 * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
13 * LOCATION: see http://www.boost.org for most recent version.
15 * VERSION see <boost/version.hpp>
16 * DESCRIPTION: Unicode regular expressions on top of the ICU Library.
18 #define BOOST_REGEX_SOURCE
20 #include <boost/regex/config.hpp>
22 #define BOOST_REGEX_ICU_INSTANTIATE
23 #include <boost/regex/icu.hpp>
29 icu_regex_traits_implementation::string_type icu_regex_traits_implementation::do_transform(const char_type* p1, const char_type* p2, const U_NAMESPACE_QUALIFIER Collator* pcoll) const
31 // TODO make thread safe!!!! :
32 typedef u32_to_u16_iterator<const char_type*, ::UChar> itt;
34 #ifndef BOOST_NO_TEMPLATED_ITERATOR_CONSTRUCTORS
35 std::vector< ::UChar> t(i, j);
37 std::vector< ::UChar> t;
41 ::uint8_t result[100];
44 len = pcoll->getSortKey(&*t.begin(), static_cast< ::int32_t>(t.size()), result, sizeof(result));
46 len = pcoll->getSortKey(static_cast<UChar const*>(0), static_cast< ::int32_t>(0), result, sizeof(result));
47 if(std::size_t(len) > sizeof(result))
49 scoped_array< ::uint8_t> presult(new ::uint8_t[len+1]);
51 len = pcoll->getSortKey(&*t.begin(), static_cast< ::int32_t>(t.size()), presult.get(), len+1);
53 len = pcoll->getSortKey(static_cast<UChar const*>(0), static_cast< ::int32_t>(0), presult.get(), len+1);
54 if((0 == presult[len-1]) && (len > 1))
56 #ifndef BOOST_NO_TEMPLATED_ITERATOR_CONSTRUCTORS
57 return string_type(presult.get(), presult.get()+len);
60 ::uint8_t const* ia = presult.get();
61 ::uint8_t const* ib = presult.get()+len;
63 sresult.push_back(*ia++);
67 if((0 == result[len-1]) && (len > 1))
69 #ifndef BOOST_NO_TEMPLATED_ITERATOR_CONSTRUCTORS
70 return string_type(result, result+len);
73 ::uint8_t const* ia = result;
74 ::uint8_t const* ib = result+len;
76 sresult.push_back(*ia++);
83 icu_regex_traits::size_type icu_regex_traits::length(const char_type* p)
95 // define our bitmasks:
97 const icu_regex_traits::char_class_type icu_regex_traits::mask_blank = icu_regex_traits::char_class_type(1) << offset_blank;
98 const icu_regex_traits::char_class_type icu_regex_traits::mask_space = icu_regex_traits::char_class_type(1) << offset_space;
99 const icu_regex_traits::char_class_type icu_regex_traits::mask_xdigit = icu_regex_traits::char_class_type(1) << offset_xdigit;
100 const icu_regex_traits::char_class_type icu_regex_traits::mask_underscore = icu_regex_traits::char_class_type(1) << offset_underscore;
101 const icu_regex_traits::char_class_type icu_regex_traits::mask_unicode = icu_regex_traits::char_class_type(1) << offset_unicode;
102 const icu_regex_traits::char_class_type icu_regex_traits::mask_any = icu_regex_traits::char_class_type(1) << offset_any;
103 const icu_regex_traits::char_class_type icu_regex_traits::mask_ascii = icu_regex_traits::char_class_type(1) << offset_ascii;
104 const icu_regex_traits::char_class_type icu_regex_traits::mask_horizontal = icu_regex_traits::char_class_type(1) << offset_horizontal;
105 const icu_regex_traits::char_class_type icu_regex_traits::mask_vertical = icu_regex_traits::char_class_type(1) << offset_vertical;
107 icu_regex_traits::char_class_type icu_regex_traits::lookup_icu_mask(const ::UChar32* p1, const ::UChar32* p2)
109 static const ::UChar32 prop_name_table[] = {
110 /* any */ 'a', 'n', 'y',
111 /* ascii */ 'a', 's', 'c', 'i', 'i',
112 /* assigned */ 'a', 's', 's', 'i', 'g', 'n', 'e', 'd',
116 /* closepunctuation */ 'c', 'l', 'o', 's', 'e', 'p', 'u', 'n', 'c', 't', 'u', 'a', 't', 'i', 'o', 'n',
119 /* connectorpunctuation */ 'c', 'o', 'n', 'n', 'e', 'c', 't', 'o', 'r', 'p', 'u', 'n', 'c', 't', 'u', 'a', 't', 'i', 'o', 'n',
120 /* control */ 'c', 'o', 'n', 't', 'r', 'o', 'l',
122 /* currencysymbol */ 'c', 'u', 'r', 'r', 'e', 'n', 'c', 'y', 's', 'y', 'm', 'b', 'o', 'l',
123 /* dashpunctuation */ 'd', 'a', 's', 'h', 'p', 'u', 'n', 'c', 't', 'u', 'a', 't', 'i', 'o', 'n',
124 /* decimaldigitnumber */ 'd', 'e', 'c', 'i', 'm', 'a', 'l', 'd', 'i', 'g', 'i', 't', 'n', 'u', 'm', 'b', 'e', 'r',
125 /* enclosingmark */ 'e', 'n', 'c', 'l', 'o', 's', 'i', 'n', 'g', 'm', 'a', 'r', 'k',
126 /* finalpunctuation */ 'f', 'i', 'n', 'a', 'l', 'p', 'u', 'n', 'c', 't', 'u', 'a', 't', 'i', 'o', 'n',
127 /* format */ 'f', 'o', 'r', 'm', 'a', 't',
128 /* initialpunctuation */ 'i', 'n', 'i', 't', 'i', 'a', 'l', 'p', 'u', 'n', 'c', 't', 'u', 'a', 't', 'i', 'o', 'n',
130 /* letter */ 'l', 'e', 't', 't', 'e', 'r',
131 /* letternumber */ 'l', 'e', 't', 't', 'e', 'r', 'n', 'u', 'm', 'b', 'e', 'r',
132 /* lineseparator */ 'l', 'i', 'n', 'e', 's', 'e', 'p', 'a', 'r', 'a', 't', 'o', 'r',
136 /* lowercaseletter */ 'l', 'o', 'w', 'e', 'r', 'c', 'a', 's', 'e', 'l', 'e', 't', 't', 'e', 'r',
140 /* mark */ 'm', 'a', 'r', 'k',
141 /* mathsymbol */ 'm', 'a', 't', 'h', 's', 'y', 'm', 'b', 'o', 'l',
145 /* modifierletter */ 'm', 'o', 'd', 'i', 'f', 'i', 'e', 'r', 'l', 'e', 't', 't', 'e', 'r',
146 /* modifiersymbol */ 'm', 'o', 'd', 'i', 'f', 'i', 'e', 'r', 's', 'y', 'm', 'b', 'o', 'l',
151 /* nonspacingmark */ 'n', 'o', 'n', 's', 'p', 'a', 'c', 'i', 'n', 'g', 'm', 'a', 'r', 'k',
152 /* notassigned */ 'n', 'o', 't', 'a', 's', 's', 'i', 'g', 'n', 'e', 'd',
153 /* number */ 'n', 'u', 'm', 'b', 'e', 'r',
154 /* openpunctuation */ 'o', 'p', 'e', 'n', 'p', 'u', 'n', 'c', 't', 'u', 'a', 't', 'i', 'o', 'n',
155 /* other */ 'o', 't', 'h', 'e', 'r',
156 /* otherletter */ 'o', 't', 'h', 'e', 'r', 'l', 'e', 't', 't', 'e', 'r',
157 /* othernumber */ 'o', 't', 'h', 'e', 'r', 'n', 'u', 'm', 'b', 'e', 'r',
158 /* otherpunctuation */ 'o', 't', 'h', 'e', 'r', 'p', 'u', 'n', 'c', 't', 'u', 'a', 't', 'i', 'o', 'n',
159 /* othersymbol */ 'o', 't', 'h', 'e', 'r', 's', 'y', 'm', 'b', 'o', 'l',
161 /* paragraphseparator */ 'p', 'a', 'r', 'a', 'g', 'r', 'a', 'p', 'h', 's', 'e', 'p', 'a', 'r', 'a', 't', 'o', 'r',
168 /* privateuse */ 'p', 'r', 'i', 'v', 'a', 't', 'e', 'u', 's', 'e',
170 /* punctuation */ 'p', 'u', 'n', 'c', 't', 'u', 'a', 't', 'i', 'o', 'n',
173 /* separator */ 's', 'e', 'p', 'a', 'r', 'a', 't', 'o', 'r',
177 /* spaceseparator */ 's', 'p', 'a', 'c', 'e', 's', 'e', 'p', 'a', 'r', 'a', 't', 'o', 'r',
178 /* spacingcombiningmark */ 's', 'p', 'a', 'c', 'i', 'n', 'g', 'c', 'o', 'm', 'b', 'i', 'n', 'i', 'n', 'g', 'm', 'a', 'r', 'k',
179 /* surrogate */ 's', 'u', 'r', 'r', 'o', 'g', 'a', 't', 'e',
180 /* symbol */ 's', 'y', 'm', 'b', 'o', 'l',
181 /* titlecase */ 't', 'i', 't', 'l', 'e', 'c', 'a', 's', 'e',
182 /* titlecaseletter */ 't', 'i', 't', 'l', 'e', 'c', 'a', 's', 'e', 'l', 'e', 't', 't', 'e', 'r',
183 /* uppercaseletter */ 'u', 'p', 'p', 'e', 'r', 'c', 'a', 's', 'e', 'l', 'e', 't', 't', 'e', 'r',
190 static const re_detail::character_pointer_range< ::UChar32> range_data[] = {
191 { prop_name_table+0, prop_name_table+3, }, // any
192 { prop_name_table+3, prop_name_table+8, }, // ascii
193 { prop_name_table+8, prop_name_table+16, }, // assigned
194 { prop_name_table+16, prop_name_table+18, }, // c*
195 { prop_name_table+18, prop_name_table+20, }, // cc
196 { prop_name_table+20, prop_name_table+22, }, // cf
197 { prop_name_table+22, prop_name_table+38, }, // closepunctuation
198 { prop_name_table+38, prop_name_table+40, }, // cn
199 { prop_name_table+40, prop_name_table+42, }, // co
200 { prop_name_table+42, prop_name_table+62, }, // connectorpunctuation
201 { prop_name_table+62, prop_name_table+69, }, // control
202 { prop_name_table+69, prop_name_table+71, }, // cs
203 { prop_name_table+71, prop_name_table+85, }, // currencysymbol
204 { prop_name_table+85, prop_name_table+100, }, // dashpunctuation
205 { prop_name_table+100, prop_name_table+118, }, // decimaldigitnumber
206 { prop_name_table+118, prop_name_table+131, }, // enclosingmark
207 { prop_name_table+131, prop_name_table+147, }, // finalpunctuation
208 { prop_name_table+147, prop_name_table+153, }, // format
209 { prop_name_table+153, prop_name_table+171, }, // initialpunctuation
210 { prop_name_table+171, prop_name_table+173, }, // l*
211 { prop_name_table+173, prop_name_table+179, }, // letter
212 { prop_name_table+179, prop_name_table+191, }, // letternumber
213 { prop_name_table+191, prop_name_table+204, }, // lineseparator
214 { prop_name_table+204, prop_name_table+206, }, // ll
215 { prop_name_table+206, prop_name_table+208, }, // lm
216 { prop_name_table+208, prop_name_table+210, }, // lo
217 { prop_name_table+210, prop_name_table+225, }, // lowercaseletter
218 { prop_name_table+225, prop_name_table+227, }, // lt
219 { prop_name_table+227, prop_name_table+229, }, // lu
220 { prop_name_table+229, prop_name_table+231, }, // m*
221 { prop_name_table+231, prop_name_table+235, }, // mark
222 { prop_name_table+235, prop_name_table+245, }, // mathsymbol
223 { prop_name_table+245, prop_name_table+247, }, // mc
224 { prop_name_table+247, prop_name_table+249, }, // me
225 { prop_name_table+249, prop_name_table+251, }, // mn
226 { prop_name_table+251, prop_name_table+265, }, // modifierletter
227 { prop_name_table+265, prop_name_table+279, }, // modifiersymbol
228 { prop_name_table+279, prop_name_table+281, }, // n*
229 { prop_name_table+281, prop_name_table+283, }, // nd
230 { prop_name_table+283, prop_name_table+285, }, // nl
231 { prop_name_table+285, prop_name_table+287, }, // no
232 { prop_name_table+287, prop_name_table+301, }, // nonspacingmark
233 { prop_name_table+301, prop_name_table+312, }, // notassigned
234 { prop_name_table+312, prop_name_table+318, }, // number
235 { prop_name_table+318, prop_name_table+333, }, // openpunctuation
236 { prop_name_table+333, prop_name_table+338, }, // other
237 { prop_name_table+338, prop_name_table+349, }, // otherletter
238 { prop_name_table+349, prop_name_table+360, }, // othernumber
239 { prop_name_table+360, prop_name_table+376, }, // otherpunctuation
240 { prop_name_table+376, prop_name_table+387, }, // othersymbol
241 { prop_name_table+387, prop_name_table+389, }, // p*
242 { prop_name_table+389, prop_name_table+407, }, // paragraphseparator
243 { prop_name_table+407, prop_name_table+409, }, // pc
244 { prop_name_table+409, prop_name_table+411, }, // pd
245 { prop_name_table+411, prop_name_table+413, }, // pe
246 { prop_name_table+413, prop_name_table+415, }, // pf
247 { prop_name_table+415, prop_name_table+417, }, // pi
248 { prop_name_table+417, prop_name_table+419, }, // po
249 { prop_name_table+419, prop_name_table+429, }, // privateuse
250 { prop_name_table+429, prop_name_table+431, }, // ps
251 { prop_name_table+431, prop_name_table+442, }, // punctuation
252 { prop_name_table+442, prop_name_table+444, }, // s*
253 { prop_name_table+444, prop_name_table+446, }, // sc
254 { prop_name_table+446, prop_name_table+455, }, // separator
255 { prop_name_table+455, prop_name_table+457, }, // sk
256 { prop_name_table+457, prop_name_table+459, }, // sm
257 { prop_name_table+459, prop_name_table+461, }, // so
258 { prop_name_table+461, prop_name_table+475, }, // spaceseparator
259 { prop_name_table+475, prop_name_table+495, }, // spacingcombiningmark
260 { prop_name_table+495, prop_name_table+504, }, // surrogate
261 { prop_name_table+504, prop_name_table+510, }, // symbol
262 { prop_name_table+510, prop_name_table+519, }, // titlecase
263 { prop_name_table+519, prop_name_table+534, }, // titlecaseletter
264 { prop_name_table+534, prop_name_table+549, }, // uppercaseletter
265 { prop_name_table+549, prop_name_table+551, }, // z*
266 { prop_name_table+551, prop_name_table+553, }, // zl
267 { prop_name_table+553, prop_name_table+555, }, // zp
268 { prop_name_table+555, prop_name_table+557, }, // zs
271 static const icu_regex_traits::char_class_type icu_class_map[] = {
272 icu_regex_traits::mask_any, // any
273 icu_regex_traits::mask_ascii, // ascii
274 (0x3FFFFFFFu) & ~(U_GC_CN_MASK), // assigned
278 U_GC_PE_MASK, // closepunctuation
281 U_GC_PC_MASK, // connectorpunctuation
282 U_GC_CC_MASK, // control
284 U_GC_SC_MASK, // currencysymbol
285 U_GC_PD_MASK, // dashpunctuation
286 U_GC_ND_MASK, // decimaldigitnumber
287 U_GC_ME_MASK, // enclosingmark
288 U_GC_PF_MASK, // finalpunctuation
289 U_GC_CF_MASK, // format
290 U_GC_PI_MASK, // initialpunctuation
292 U_GC_L_MASK, // letter
293 U_GC_NL_MASK, // letternumber
294 U_GC_ZL_MASK, // lineseparator
298 U_GC_LL_MASK, // lowercaseletter
303 U_GC_SM_MASK, // mathsymbol
307 U_GC_LM_MASK, // modifierletter
308 U_GC_SK_MASK, // modifiersymbol
313 U_GC_MN_MASK, // nonspacingmark
314 U_GC_CN_MASK, // notassigned
315 U_GC_N_MASK, // number
316 U_GC_PS_MASK, // openpunctuation
317 U_GC_C_MASK, // other
318 U_GC_LO_MASK, // otherletter
319 U_GC_NO_MASK, // othernumber
320 U_GC_PO_MASK, // otherpunctuation
321 U_GC_SO_MASK, // othersymbol
323 U_GC_ZP_MASK, // paragraphseparator
330 U_GC_CO_MASK, // privateuse
332 U_GC_P_MASK, // punctuation
335 U_GC_Z_MASK, // separator
339 U_GC_ZS_MASK, // spaceseparator
340 U_GC_MC_MASK, // spacingcombiningmark
341 U_GC_CS_MASK, // surrogate
342 U_GC_S_MASK, // symbol
343 U_GC_LT_MASK, // titlecase
344 U_GC_LT_MASK, // titlecaseletter
345 U_GC_LU_MASK, // uppercaseletter
353 static const re_detail::character_pointer_range< ::UChar32>* ranges_begin = range_data;
354 static const re_detail::character_pointer_range< ::UChar32>* ranges_end = range_data + (sizeof(range_data)/sizeof(range_data[0]));
356 re_detail::character_pointer_range< ::UChar32> t = { p1, p2, };
357 const re_detail::character_pointer_range< ::UChar32>* p = std::lower_bound(ranges_begin, ranges_end, t);
358 if((p != ranges_end) && (t == *p))
359 return icu_class_map[p - ranges_begin];
363 icu_regex_traits::char_class_type icu_regex_traits::lookup_classname(const char_type* p1, const char_type* p2) const
365 static const char_class_type masks[] =
368 U_GC_L_MASK | U_GC_ND_MASK,
371 U_GC_CC_MASK | U_GC_CF_MASK | U_GC_ZL_MASK | U_GC_ZP_MASK,
374 (0x3FFFFFFFu) & ~(U_GC_CC_MASK | U_GC_CF_MASK | U_GC_CS_MASK | U_GC_CN_MASK | U_GC_Z_MASK),
380 char_class_type(U_GC_Z_MASK) | mask_space,
381 char_class_type(U_GC_Z_MASK) | mask_space,
386 char_class_type(U_GC_L_MASK | U_GC_ND_MASK | U_GC_MN_MASK) | mask_underscore,
387 char_class_type(U_GC_L_MASK | U_GC_ND_MASK | U_GC_MN_MASK) | mask_underscore,
388 char_class_type(U_GC_ND_MASK) | mask_xdigit,
391 int id = ::boost::re_detail::get_default_class_id(p1, p2);
394 char_class_type result = lookup_icu_mask(p1, p2);
400 string_type s(p1, p2);
401 string_type::size_type i = 0;
404 s[i] = static_cast<char>((::u_tolower)(s[i]));
405 if(::u_isspace(s[i]) || (s[i] == '-') || (s[i] == '_'))
406 s.erase(s.begin()+i, s.begin()+i+1);
409 s[i] = static_cast<char>((::u_tolower)(s[i]));
414 id = ::boost::re_detail::get_default_class_id(&*s.begin(), &*s.begin() + s.size());
418 result = lookup_icu_mask(&*s.begin(), &*s.begin() + s.size());
422 BOOST_ASSERT(std::size_t(id+1) < sizeof(masks) / sizeof(masks[0]));
426 icu_regex_traits::string_type icu_regex_traits::lookup_collatename(const char_type* p1, const char_type* p2) const
429 if(std::find_if(p1, p2, std::bind2nd(std::greater< ::UChar32>(), 0x7f)) == p2)
431 #ifndef BOOST_NO_TEMPLATED_ITERATOR_CONSTRUCTORS
432 std::string s(p1, p2);
435 const char_type* p3 = p1;
440 UErrorCode err = U_ZERO_ERROR;
441 UChar32 c = ::u_charFromName(U_UNICODE_CHAR_NAME, s.c_str(), &err);
447 // Try Unicode-extended name:
449 c = ::u_charFromName(U_EXTENDED_CHAR_NAME, s.c_str(), &err);
456 s = ::boost::re_detail::lookup_default_collate_name(s);
457 #ifndef BOOST_NO_TEMPLATED_ITERATOR_CONSTRUCTORS
458 result.assign(s.begin(), s.end());
461 std::string::const_iterator si, sj;
465 result.push_back(*si++);
468 if(result.empty() && (p2-p1 == 1))
469 result.push_back(*p1);
473 bool icu_regex_traits::isctype(char_type c, char_class_type f) const
475 // check for standard catagories first:
476 char_class_type m = char_class_type(1u << u_charType(c));
479 // now check for special cases:
480 if(((f & mask_blank) != 0) && u_isblank(c))
482 if(((f & mask_space) != 0) && u_isspace(c))
484 if(((f & mask_xdigit) != 0) && (u_digit(c, 16) >= 0))
486 if(((f & mask_unicode) != 0) && (c >= 0x100))
488 if(((f & mask_underscore) != 0) && (c == '_'))
490 if(((f & mask_any) != 0) && (c <= 0x10FFFF))
492 if(((f & mask_ascii) != 0) && (c <= 0x7F))
494 if(((f & mask_vertical) != 0) && (::boost::re_detail::is_separator(c) || (c == static_cast<char_type>('\v')) || (m == U_GC_ZL_MASK) || (m == U_GC_ZP_MASK)))
496 if(((f & mask_horizontal) != 0) && !::boost::re_detail::is_separator(c) && u_isspace(c) && (c != static_cast<char_type>('\v')))
503 #endif // BOOST_HAS_ICU