+ //===========================================================================
+ // The classes here are used by offset_separator and char_separator to implement
+ // faster assigning of tokens using assign instead of +=
+
+ namespace tokenizer_detail {
+ //===========================================================================
+ // Tokenizer was broken for wide character separators, at least on Windows, since
+ // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts
+ // if higher values are passed in. The traits extension class should take care of this.
+ // Assuming that the conditional will always get optimized out in the function
+ // implementations, argument types are not a problem since both forms of character classifiers
+ // expect an int.
+
+#if !defined(BOOST_NO_CWCTYPE)
+ template<typename traits, int N>
+ struct traits_extension_details : public traits {
+ typedef typename traits::char_type char_type;
+ static bool isspace(char_type c)
+ {
+ return std::iswspace(c) != 0;
+ }
+ static bool ispunct(char_type c)
+ {
+ return std::iswpunct(c) != 0;
+ }
+ };
+
+ template<typename traits>
+ struct traits_extension_details<traits, 1> : public traits {
+ typedef typename traits::char_type char_type;
+ static bool isspace(char_type c)
+ {
+ return std::isspace(c) != 0;
+ }
+ static bool ispunct(char_type c)
+ {
+ return std::ispunct(c) != 0;
+ }
+ };
+#endif
+
+
+ // In case there is no cwctype header, we implement the checks manually.
+ // We make use of the fact that the tested categories should fit in ASCII.
+ template<typename traits>
+ struct traits_extension : public traits {
+ typedef typename traits::char_type char_type;
+ static bool isspace(char_type c)
+ {
+#if !defined(BOOST_NO_CWCTYPE)
+ return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
+#else
+ return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
+#endif
+ }
+
+ static bool ispunct(char_type c)
+ {
+#if !defined(BOOST_NO_CWCTYPE)
+ return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
+#else
+ return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
+#endif
+ }
+ };
+
+ // The assign_or_plus_equal struct contains functions that implement
+ // assign, +=, and clearing based on the iterator type. The
+ // generic case does nothing for plus_equal and clearing, while
+ // passing through the call for assign.
+ //
+ // When an input iterator is being used, the situation is reversed.
+ // The assign method does nothing, plus_equal invokes operator +=,
+ // and the clearing method sets the supplied token to the default
+ // token constructor's result.
+ //
+
+ template<class IteratorTag>
+ struct assign_or_plus_equal {
+ template<class Iterator, class Token>
+ static void assign(Iterator b, Iterator e, Token &t) {
+ t.assign(b, e);
+ }
+
+ template<class Token, class Value>
+ static void plus_equal(Token &, const Value &) { }
+
+ // If we are doing an assign, there is no need for the
+ // the clear.
+ //
+ template<class Token>
+ static void clear(Token &) { }
+ };
+
+ template <>
+ struct assign_or_plus_equal<std::input_iterator_tag> {
+ template<class Iterator, class Token>
+ static void assign(Iterator , Iterator , Token &) { }
+ template<class Token, class Value>
+ static void plus_equal(Token &t, const Value &v) {
+ t += v;
+ }
+ template<class Token>
+ static void clear(Token &t) {
+ t = Token();
+ }
+ };
+
+
+ template<class Iterator>
+ struct pointer_iterator_category{
+ typedef std::random_access_iterator_tag type;
+ };
+
+
+ template<class Iterator>
+ struct class_iterator_category{
+ typedef typename Iterator::iterator_category type;
+ };
+
+
+
+ // This portably gets the iterator_tag without partial template specialization
+ template<class Iterator>
+ struct get_iterator_category{
+ typedef typename mpl::if_<is_pointer<Iterator>,
+ pointer_iterator_category<Iterator>,
+ class_iterator_category<Iterator>
+ >::type cat;
+
+ typedef typename cat::type iterator_category;
+ };
+
+
+ } // namespace tokenizer_detail
+
+