+ //===========================================================================
+ // Tokenizer was broken for wide character separators, at least on Windows, since
+ // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts
+ // if higher values are passed in. The traits extension class should take care of this.
+ // Assuming that the conditional will always get optimized out in the function
+ // implementations, argument types are not a problem since both forms of character classifiers
+ // expect an int.
+
+#if !defined(BOOST_NO_CWCTYPE)
+ template<typename traits, int N>
+ struct traits_extension_details : public traits {
+ typedef typename traits::char_type char_type;
+ static bool isspace(char_type c)
+ {
+ return std::iswspace(c) != 0;
+ }
+ static bool ispunct(char_type c)
+ {
+ return std::iswpunct(c) != 0;
+ }
+ };
+
+ template<typename traits>
+ struct traits_extension_details<traits, 1> : public traits {
+ typedef typename traits::char_type char_type;
+ static bool isspace(char_type c)
+ {
+ return std::isspace(c) != 0;
+ }
+ static bool ispunct(char_type c)
+ {
+ return std::ispunct(c) != 0;
+ }
+ };
+#endif
+
+
+ // In case there is no cwctype header, we implement the checks manually.
+ // We make use of the fact that the tested categories should fit in ASCII.
+ template<typename traits>
+ struct traits_extension : public traits {
+ typedef typename traits::char_type char_type;
+ static bool isspace(char_type c)
+ {
+#if !defined(BOOST_NO_CWCTYPE)
+ return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
+#else
+ return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
+#endif
+ }
+
+ static bool ispunct(char_type c)
+ {
+#if !defined(BOOST_NO_CWCTYPE)
+ return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
+#else
+ return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
+#endif
+ }
+ };