1 // Boost tokenizer.hpp -----------------------------------------------------//
3 // © Copyright Jeremy Siek and John R. Bandela 2001.
5 // Permission to copy, use, modify, sell and distribute this software
6 // is granted provided this copyright notice appears in all
7 // copies. This software is provided "as is" without express or
8 // implied warranty, and with no claim as to its suitability for any
11 // See http://www.boost.org for updates, documentation, and revision history.
15 // 02 Feb 2002 Jeremy Siek
16 // Removed tabs and a little cleanup.
18 #ifndef BOOST_TOKENIZER_JRB051801_HPP_
19 #define BOOST_TOKENIZER_JRB051801_HPP_
21 #include <boost/token_iterator.hpp>
26 //===========================================================================
27 // A container-view of a tokenized "sequence"
29 typename TokenizerFunc = char_delimiters_separator<char>,
30 typename Iterator = std::string::const_iterator,
31 typename Type = std::string
35 typedef detail::tokenizer_policy<Type, TokenizerFunc> Pol;
36 typedef detail::token_iterator_base<Iterator> TBase;
37 typedef token_iterator_generator<TokenizerFunc,Iterator,Type> TGen;
39 // It seems that MSVC does not like the unqualified use of iterator,
40 // Thus we use iter internally when it is used unqualified and
41 // the users of this class will always qualify iterator.
42 typedef typename TGen::type iter;
46 typedef iter iterator;
47 typedef iter const_iterator;
48 typedef Type value_type;
49 typedef value_type& reference;
50 typedef const value_type& const_reference;
51 typedef value_type* pointer;
52 typedef const pointer const_pointer;
53 typedef void size_type;
54 typedef void difference_type;
56 tokenizer(Iterator first, Iterator last,
57 const TokenizerFunc& f = TokenizerFunc())
58 : first_(first), last_(last), f_(f) { }
60 template <typename Container>
61 tokenizer(const Container& c)
62 : first_(c.begin()), last_(c.end()), f_() { }
64 template <typename Container>
65 tokenizer(const Container& c,const TokenizerFunc& f)
66 : first_(c.begin()), last_(c.end()), f_(f) { }
68 void assign(Iterator first, Iterator last){
73 void assign(Iterator first, Iterator last, const TokenizerFunc& f){
78 template <typename Container>
79 void assign(const Container& c){
80 assign(c.begin(),c.end());
84 template <typename Container>
85 void assign(const Container& c, const TokenizerFunc& f){
86 assign(c.begin(),c.end(),f);
89 iter begin() const { return iter(TBase(first_,last_),Pol(f_)); }
90 iter end() const { return iter(TBase(last_,last_),Pol(f_)); }