+#if !defined(BOOST_NO_CWCTYPE)
+ template<typename traits, int N>
+ struct traits_extension_details : public traits {
+ typedef typename traits::char_type char_type;
+ static bool isspace(char_type c)
+ {
+ return std::iswspace(c) != 0;
+ }
+ static bool ispunct(char_type c)
+ {
+ return std::iswpunct(c) != 0;
+ }
+ };
+
+ template<typename traits>
+ struct traits_extension_details<traits, 1> : public traits {
+ typedef typename traits::char_type char_type;
+ static bool isspace(char_type c)
+ {
+ return std::isspace(c) != 0;
+ }
+ static bool ispunct(char_type c)
+ {
+ return std::ispunct(c) != 0;
+ }
+ };
+#endif
+
+
+ // In case there is no cwctype header, we implement the checks manually.
+ // We make use of the fact that the tested categories should fit in ASCII.
+ template<typename traits>
+ struct traits_extension : public traits {
+ typedef typename traits::char_type char_type;
+ static bool isspace(char_type c)
+ {
+#if !defined(BOOST_NO_CWCTYPE)
+ return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
+#else
+ return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
+#endif
+ }
+
+ static bool ispunct(char_type c)
+ {
+#if !defined(BOOST_NO_CWCTYPE)
+ return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
+#else
+ return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
+#endif
+ }
+ };
+
+ // The assign_or_plus_equal struct contains functions that implement
+ // assign, +=, and clearing based on the iterator type. The
+ // generic case does nothing for plus_equal and clearing, while
+ // passing through the call for assign.
+ //
+ // When an input iterator is being used, the situation is reversed.
+ // The assign method does nothing, plus_equal invokes operator +=,
+ // and the clearing method sets the supplied token to the default
+ // token constructor's result.
+ //
+
+ template<class IteratorTag>
+ struct assign_or_plus_equal {
+ template<class Iterator, class Token>
+ static void assign(Iterator b, Iterator e, Token &t) {
+
+#if BOOST_WORKAROUND(BOOST_MSVC, < 1300) &&\
+ BOOST_WORKAROUND(__SGI_STL_PORT, < 0x500) &&\
+ defined(_STLP_DEBUG) &&\
+ (defined(_STLP_USE_DYNAMIC_LIB) || defined(_DLL))
+ // Problem with string::assign for msvc-stlport in debug mode: the
+ // linker tries to import the templatized version of this memfun,
+ // which is obviously not exported.
+ // See http://www.stlport.com/dcforum/DCForumID6/1763.html for details.
+
+ t = Token();
+ while(b != e) t += *b++;
+#else
+ t.assign(b, e);
+#endif
+
+ }
+
+ template<class Token, class Value>
+ static void plus_equal(Token &, const Value &) { }
+
+ // If we are doing an assign, there is no need for the
+ // the clear.
+ //
+ template<class Token>
+ static void clear(Token &) { }
+ };
+
+ template <>
+ struct assign_or_plus_equal<std::input_iterator_tag> {
+ template<class Iterator, class Token>
+ static void assign(Iterator b, Iterator e, Token &t) { }
+ template<class Token, class Value>
+ static void plus_equal(Token &t, const Value &v) {
+ t += v;
+ }
+ template<class Token>
+ static void clear(Token &t) {
+ t = Token();
+ }
+ };
+
+
+ template<class Iterator>
+ struct pointer_iterator_category{
+ typedef std::random_access_iterator_tag type;
+ };
+
+
+ template<class Iterator>
+ struct class_iterator_category{
+ typedef typename Iterator::iterator_category type;
+ };
+
+
+
+ // This portably gets the iterator_tag without partial template specialization
+ template<class Iterator>
+ struct get_iterator_category{
+ typedef typename mpl::if_<is_pointer<Iterator>,
+ pointer_iterator_category<Iterator>,
+ class_iterator_category<Iterator>
+ >::type cat;
+
+ typedef typename cat::type iterator_category;
+ };
+
+
+ } // namespace tokenizer_detail
+
+