1 // Boost token_functions.hpp ------------------------------------------------//
3 // Copyright John R. Bandela 2001.
5 // Distributed under the Boost Software License, Version 1.0. (See
6 // accompanying file LICENSE_1_0.txt or copy at
7 // http://www.boost.org/LICENSE_1_0.txt)
9 // See http://www.boost.org/libs/tokenizer/ for documentation.
12 // 01 Oct 2004 Joaquín M López Muñoz
13 // Workaround for a problem with string::assign in msvc-stlport
14 // 06 Apr 2004 John Bandela
15 // Fixed a bug involving using char_delimiter with a true input iterator
16 // 28 Nov 2003 Robert Zeh and John Bandela
17 // Converted into "fast" functions that avoid using += when
18 // the supplied iterator isn't an input_iterator; based on
19 // some work done at Archelon and a version that was checked into
20 // the boost CVS for a short period of time.
21 // 20 Feb 2002 John Maddock
22 // Removed using namespace std declarations and added
23 // workaround for BOOST_NO_STDC_NAMESPACE (the library
24 // can be safely mixed with regex).
25 // 06 Feb 2002 Jeremy Siek
26 // Added char_separator.
27 // 02 Feb 2002 Jeremy Siek
28 // Removed tabs and a little cleanup.
31 #ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
32 #define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
38 #include <algorithm> // for find_if
39 #include <boost/config.hpp>
40 #include <boost/assert.hpp>
41 #include <boost/detail/workaround.hpp>
42 #include <boost/mpl/if.hpp>
45 // the following must not be macros if we are to prefix them
46 // with std:: (they shouldn't be macros anyway...)
55 // fix namespace problems:
57 #ifdef BOOST_NO_STDC_NAMESPACE
66 //===========================================================================
67 // The escaped_list_separator class. Which is a model of TokenizerFunction
68 // An escaped list is a super-set of what is commonly known as a comma
69 // separated value (csv) list.It is separated into fields by a comma or
70 // other character. If the delimiting character is inside quotes, then it is
71 // counted as a regular character.To allow for embedded quotes in a field,
72 // there can be escape sequences using the \ much like C.
73 // The role of the comma, the quotation mark, and the escape
74 // character (backslash \), can be assigned to other characters.
76 struct escaped_list_error : public std::runtime_error{
77 escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
81 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
82 // MSVC does not like the following typename
83 #if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
85 class Traits = typename std::basic_string<Char>::traits_type >
88 class Traits = std::basic_string<Char>::traits_type >
90 class escaped_list_separator {
93 typedef std::basic_string<Char,Traits> string_type;
96 char_eq(Char e):e_(e) { }
97 bool operator()(Char c) {
98 return Traits::eq(e_,c);
106 bool is_escape(Char e) {
108 return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
112 return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
114 bool is_quote(Char e) {
116 return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
118 template <typename iterator, typename Token>
119 void do_escape(iterator& next,iterator end,Token& tok) {
121 throw escaped_list_error(std::string("cannot end with escape"));
122 if (Traits::eq(*next,'n')) {
126 else if (is_quote(*next)) {
130 else if (is_c(*next)) {
134 else if (is_escape(*next)) {
139 throw escaped_list_error(std::string("unknown escape sequence"));
144 explicit escaped_list_separator(Char e = '\\',
145 Char c = ',',Char q = '\"')
146 : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
148 escaped_list_separator(string_type e, string_type c, string_type q)
149 : escape_(e), c_(c), quote_(q), last_(false) { }
151 void reset() {last_=false;}
153 template <typename InputIterator, typename Token>
154 bool operator()(InputIterator& next,InputIterator end,Token& tok) {
155 bool bInQuote = false;
167 for (;next != end;++next) {
168 if (is_escape(*next)) {
169 do_escape(next,end,tok);
171 else if (is_c(*next)) {
173 // If we are not in quote, then we are done
175 // The last character was a c, that means there is
176 // 1 more blank field
182 else if (is_quote(*next)) {
193 //===========================================================================
194 // The classes here are used by offset_separator and char_separator to implement
195 // faster assigning of tokens using assign instead of +=
197 namespace tokenizer_detail {
199 // The assign_or_plus_equal struct contains functions that implement
200 // assign, +=, and clearing based on the iterator type. The
201 // generic case does nothing for plus_equal and clearing, while
202 // passing through the call for assign.
204 // When an input iterator is being used, the situation is reversed.
205 // The assign method does nothing, plus_equal invokes operator +=,
206 // and the clearing method sets the supplied token to the default
207 // token constructor's result.
210 template<class IteratorTag>
211 struct assign_or_plus_equal {
212 template<class Iterator, class Token>
213 static void assign(Iterator b, Iterator e, Token &t) {
215 #if BOOST_WORKAROUND(BOOST_MSVC, < 1300) &&\
216 BOOST_WORKAROUND(__SGI_STL_PORT, < 0x500) &&\
217 defined(_STLP_DEBUG) &&\
218 (defined(_STLP_USE_DYNAMIC_LIB) || defined(_DLL))
219 // Problem with string::assign for msvc-stlport in debug mode: the
220 // linker tries to import the templatized version of this memfun,
221 // which is obviously not exported.
222 // See http://www.stlport.com/dcforum/DCForumID6/1763.html for details.
225 while(b != e) t += *b++;
232 template<class Token, class Value>
233 static void plus_equal(Token &, const Value &) {
237 // If we are doing an assign, there is no need for the
240 template<class Token>
241 static void clear(Token &) {
247 struct assign_or_plus_equal<std::input_iterator_tag> {
248 template<class Iterator, class Token>
249 static void assign(Iterator b, Iterator e, Token &t) {
252 template<class Token, class Value>
253 static void plus_equal(Token &t, const Value &v) {
256 template<class Token>
257 static void clear(Token &t) {
263 template<class Iterator>
264 struct pointer_iterator_category{
265 typedef std::random_access_iterator_tag type;
269 template<class Iterator>
270 struct class_iterator_category{
271 typedef typename Iterator::iterator_category type;
276 // This portably gets the iterator_tag without partial template specialization
277 template<class Iterator>
278 struct get_iterator_category{
279 typedef typename mpl::if_<is_pointer<Iterator>,
280 pointer_iterator_category<Iterator>,
281 class_iterator_category<Iterator>
284 typedef typename cat::type iterator_category;
291 //===========================================================================
292 // The offset_separator class, which is a model of TokenizerFunction.
293 // Offset breaks a string into tokens based on a range of offsets
295 class offset_separator {
298 std::vector<int> offsets_;
299 unsigned int current_offset_;
301 bool return_partial_last_;
304 template <typename Iter>
305 offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
306 bool return_partial_last = true)
307 : offsets_(begin,end), current_offset_(0),
308 wrap_offsets_(wrap_offsets),
309 return_partial_last_(return_partial_last) { }
312 : offsets_(1,1), current_offset_(),
313 wrap_offsets_(true), return_partial_last_(true) { }
319 template <typename InputIterator, typename Token>
320 bool operator()(InputIterator& next, InputIterator end, Token& tok)
322 typedef tokenizer_detail::assign_or_plus_equal<
323 #if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
326 tokenizer_detail::get_iterator_category<
327 InputIterator>::iterator_category> assigner;
330 BOOST_ASSERT(!offsets_.empty());
332 assigner::clear(tok);
333 InputIterator start(next);
338 if (current_offset_ == offsets_.size()) {
345 int c = offsets_[current_offset_];
348 if (next == end)break;
349 assigner::plus_equal(tok,*next++);
351 assigner::assign(start,next,tok);
353 if (!return_partial_last_)
363 //===========================================================================
364 // The char_separator class breaks a sequence of characters into
365 // tokens based on the character delimiters (very much like bad old
366 // strtok). A delimiter character can either be kept or dropped. A
367 // kept delimiter shows up as an output token, whereas a dropped
368 // delimiter does not.
370 // This class replaces the char_delimiters_separator class. The
371 // constructor for the char_delimiters_separator class was too
372 // confusing and needed to be deprecated. However, because of the
373 // default arguments to the constructor, adding the new constructor
374 // would cause ambiguity, so instead I deprecated the whole class.
375 // The implementation of the class was also simplified considerably.
377 enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
379 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
380 #if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
381 template <typename Char,
382 typename Traits = typename std::basic_string<Char>::traits_type >
384 template <typename Char,
385 typename Traits = std::basic_string<Char>::traits_type >
389 typedef std::basic_string<Char,Traits> string_type;
392 char_separator(const Char* dropped_delims,
393 const Char* kept_delims = 0,
394 empty_token_policy empty_tokens = drop_empty_tokens)
395 : m_dropped_delims(dropped_delims),
396 m_use_ispunct(false),
397 m_use_isspace(false),
398 m_empty_tokens(empty_tokens),
401 // Borland workaround
403 m_kept_delims = kept_delims;
406 // use ispunct() for kept delimiters and isspace for dropped.
409 : m_use_ispunct(true),
411 m_empty_tokens(drop_empty_tokens) { }
415 template <typename InputIterator, typename Token>
416 bool operator()(InputIterator& next, InputIterator end, Token& tok)
418 typedef tokenizer_detail::assign_or_plus_equal<
419 #if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
422 tokenizer_detail::get_iterator_category<
423 InputIterator>::iterator_category> assigner;
425 assigner::clear(tok);
427 // skip past all dropped_delims
428 if (m_empty_tokens == drop_empty_tokens)
429 for (; next != end && is_dropped(*next); ++next)
432 InputIterator start(next);
434 if (m_empty_tokens == drop_empty_tokens) {
440 // if we are on a kept_delims move past it and stop
441 if (is_kept(*next)) {
442 assigner::plus_equal(tok,*next);
445 // append all the non delim characters
446 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
447 assigner::plus_equal(tok,*next);
449 else { // m_empty_tokens == keep_empty_tokens
451 // Handle empty token at the end
453 if (m_output_done == false) {
454 m_output_done = true;
455 assigner::assign(start,next,tok);
461 if (is_kept(*next)) {
462 if (m_output_done == false)
463 m_output_done = true;
465 assigner::plus_equal(tok,*next);
467 m_output_done = false;
470 else if (m_output_done == false && is_dropped(*next)) {
471 m_output_done = true;
474 if (is_dropped(*next))
476 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
477 assigner::plus_equal(tok,*next);
478 m_output_done = true;
481 assigner::assign(start,next,tok);
486 string_type m_kept_delims;
487 string_type m_dropped_delims;
490 empty_token_policy m_empty_tokens;
493 bool is_kept(Char E) const
495 if (m_kept_delims.length())
496 return m_kept_delims.find(E) != string_type::npos;
497 else if (m_use_ispunct) {
498 return std::ispunct(E) != 0;
502 bool is_dropped(Char E) const
504 if (m_dropped_delims.length())
505 return m_dropped_delims.find(E) != string_type::npos;
506 else if (m_use_isspace) {
507 return std::isspace(E) != 0;
513 //===========================================================================
514 // The following class is DEPRECATED, use class char_separators instead.
516 // The char_delimiters_separator class, which is a model of
517 // TokenizerFunction. char_delimiters_separator breaks a string
518 // into tokens based on character delimiters. There are 2 types of
519 // delimiters. returnable delimiters can be returned as
520 // tokens. These are often punctuation. nonreturnable delimiters
521 // cannot be returned as tokens. These are often whitespace
523 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
524 #if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
525 template <class Char,
526 class Traits = typename std::basic_string<Char>::traits_type >
528 template <class Char,
529 class Traits = std::basic_string<Char>::traits_type >
531 class char_delimiters_separator {
534 typedef std::basic_string<Char,Traits> string_type;
535 string_type returnable_;
536 string_type nonreturnable_;
541 bool is_ret(Char E)const
543 if (returnable_.length())
544 return returnable_.find(E) != string_type::npos;
546 if (no_ispunct_) {return false;}
548 int r = std::ispunct(E);
553 bool is_nonret(Char E)const
555 if (nonreturnable_.length())
556 return nonreturnable_.find(E) != string_type::npos;
558 if (no_isspace_) {return false;}
560 int r = std::isspace(E);
567 explicit char_delimiters_separator(bool return_delims = false,
568 const Char* returnable = 0,
569 const Char* nonreturnable = 0)
570 : returnable_(returnable ? returnable : string_type().c_str()),
571 nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
572 return_delims_(return_delims), no_ispunct_(returnable!=0),
573 no_isspace_(nonreturnable!=0) { }
579 template <typename InputIterator, typename Token>
580 bool operator()(InputIterator& next, InputIterator end,Token& tok) {
583 // skip past all nonreturnable delims
584 // skip past the returnable only if we are not returning delims
585 for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
586 && !return_delims_ ) );++next) { }
592 // if we are to return delims and we are one a returnable one
593 // move past it and stop
594 if (is_ret(*next) && return_delims_) {
599 // append all the non delim characters
600 for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)