1 // Boost token_functions.hpp ------------------------------------------------//
3 // Copyright John R. Bandela 2001.
5 // Permission to copy, use, modify, sell and distribute this software
6 // is granted provided this copyright notice appears in all
7 // copies. This software is provided "as is" without express or
8 // implied warranty, and with no claim as to its suitability for any
11 // See http://www.boost.org/libs/tokenizer for documentation.
15 // 20 Feb 2002 John Maddock
16 // Removed using namespace std declarations and added
17 // workaround for BOOST_NO_STDC_NAMESPACE (the library
18 // can be safely mixed with regex).
19 // 06 Feb 2002 Jeremy Siek
20 // Added char_separator.
21 // 02 Feb 2002 Jeremy Siek
22 // Removed tabs and a little cleanup.
25 #ifndef BOOST_TOKEN_FUNCTIONS_JRB051801_HPP_
26 #define BOOST_TOKEN_FUNCTIONS_JRB051801_HPP_
33 #include <algorithm> // for find_if
36 // the following must not be macros if we are to prefix them
37 // with std:: (they shouldn't be macros anyway...)
46 // fix namespace problems:
48 #ifdef BOOST_NO_STDC_NAMESPACE
57 //===========================================================================
58 // The escaped_list_separator class. Which is a model of TokenizerFunction
59 // An escaped list is a super-set of what is commonly known as a comma
60 // separated value (csv) list.It is separated into fields by a comma or
61 // other character. If the delimiting character is inside quotes, then it is
62 // counted as a regular character.To allow for embedded quotes in a field,
63 // there can be escape sequences using the \ much like C.
64 // The role of the comma, the quotation mark, and the escape
65 // character (backslash \), can be assigned to other characters.
67 struct escaped_list_error : public std::runtime_error{
68 escaped_list_error(const std::string& what):std::runtime_error(what) { }
72 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
73 // MSVC does not like the following typename
74 #if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
76 class Traits = typename std::basic_string<Char>::traits_type >
79 class Traits = std::basic_string<Char>::traits_type >
81 class escaped_list_separator {
84 typedef std::basic_string<Char,Traits> string_type;
87 char_eq(Char e):e_(e) { }
88 bool operator()(Char c) {
89 return Traits::eq(e_,c);
97 bool is_escape(Char e) {
99 return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
103 return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
105 bool is_quote(Char e) {
107 return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
109 template <typename iterator, typename Token>
110 void do_escape(iterator& next,iterator end,Token& tok) {
112 throw escaped_list_error(std::string("cannot end with escape"));
113 if (Traits::eq(*next,'n')) {
117 else if (is_quote(*next)) {
121 else if (is_c(*next)) {
125 else if (is_escape(*next)) {
130 throw escaped_list_error(std::string("unknown escape sequence"));
135 explicit escaped_list_separator(Char e = '\\',
136 Char c = ',',Char q = '\"')
137 : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
139 escaped_list_separator(string_type e, string_type c, string_type q)
140 : escape_(e), c_(c), quote_(q), last_(false) { }
142 void reset() {last_=false;}
144 template <typename InputIterator, typename Token>
145 bool operator()(InputIterator& next,InputIterator end,Token& tok) {
146 bool bInQuote = false;
158 for (;next != end;++next) {
159 if (is_escape(*next)) {
160 do_escape(next,end,tok);
162 else if (is_c(*next)) {
164 // If we are not in quote, then we are done
166 // The last character was a c, that means there is
167 // 1 more blank field
173 else if (is_quote(*next)) {
185 //===========================================================================
186 // The offset_separator class, which is a model of TokenizerFunction.
187 // Offset breaks a string into tokens based on a range of offsets
189 class offset_separator {
192 std::vector<int> offsets_;
193 unsigned int current_offset_;
195 bool return_partial_last_;
198 template <typename Iter>
199 offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
200 bool return_partial_last = true)
201 : offsets_(begin,end), current_offset_(0),
202 wrap_offsets_(wrap_offsets),
203 return_partial_last_(return_partial_last) { }
206 : offsets_(1,1), current_offset_(),
207 wrap_offsets_(true), return_partial_last_(true) { }
213 template <typename InputIterator, typename Token>
214 bool operator()(InputIterator& next, InputIterator end, Token& tok)
216 assert(!offsets_.empty());
223 if (current_offset_ == offsets_.size())
229 int c = offsets_[current_offset_];
232 if (next == end)break;
236 if (!return_partial_last_)
246 //===========================================================================
247 // The char_separator class breaks a sequence of characters into
248 // tokens based on the character delimiters (very much like bad old
249 // strtok). A delimiter character can either be kept or dropped. A
250 // kept delimiter shows up as an output token, whereas a dropped
251 // delimiter does not.
253 // This class replaces the char_delimiters_separator class. The
254 // constructor for the char_delimiters_separator class was too
255 // confusing and needed to be deprecated. However, because of the
256 // default arguments to the constructor, adding the new constructor
257 // would cause ambiguity, so instead I deprecated the whole class.
258 // The implementation of the class was also simplified considerably.
260 enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
262 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
263 #if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
264 template <typename Char,
265 typename Traits = typename std::basic_string<Char>::traits_type >
267 template <typename Char,
268 typename Traits = std::basic_string<Char>::traits_type >
272 typedef std::basic_string<Char,Traits> string_type;
275 char_separator(const Char* dropped_delims,
276 const Char* kept_delims = 0,
277 empty_token_policy empty_tokens = drop_empty_tokens)
278 : m_dropped_delims(dropped_delims),
279 m_use_ispunct(false),
280 m_use_isspace(false),
281 m_empty_tokens(empty_tokens),
284 // Borland workaround
286 m_kept_delims = kept_delims;
289 // use ispunct() for kept delimiters and isspace for dropped.
292 : m_use_ispunct(true),
294 m_empty_tokens(drop_empty_tokens) { }
298 template <typename InputIterator, typename Token>
299 bool operator()(InputIterator& next, InputIterator end, Token& tok)
303 // skip past all dropped_delims
304 if (m_empty_tokens == drop_empty_tokens)
305 for (; next != end && is_dropped(*next); ++next)
308 if (m_empty_tokens == drop_empty_tokens) {
313 // if we are on a kept_delims move past it and stop
314 if (is_kept(*next)) {
318 // append all the non delim characters
319 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
322 else { // m_empty_tokens == keep_empty_tokens
324 // Handle empty token at the end
326 if (m_output_done == false) {
327 m_output_done = true;
332 if (is_kept(*next)) {
333 if (m_output_done == false)
334 m_output_done = true;
338 m_output_done = false;
341 else if (m_output_done == false && is_dropped(*next)) {
342 m_output_done = true;
345 if (is_dropped(*next))
347 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
349 m_output_done = true;
356 string_type m_kept_delims;
357 string_type m_dropped_delims;
360 empty_token_policy m_empty_tokens;
363 bool is_kept(Char E) const
365 if (m_kept_delims.length())
366 return m_kept_delims.find(E) != string_type::npos;
367 else if (m_use_ispunct) {
368 return std::ispunct(E) != 0;
372 bool is_dropped(Char E) const
374 if (m_dropped_delims.length())
375 return m_dropped_delims.find(E) != string_type::npos;
376 else if (m_use_isspace) {
377 return std::isspace(E) != 0;
383 //===========================================================================
384 // The following class is DEPRECATED, use class char_separators instead.
386 // The char_delimiters_separator class, which is a model of
387 // TokenizerFunction. char_delimiters_separator breaks a string
388 // into tokens based on character delimiters. There are 2 types of
389 // delimiters. returnable delimiters can be returned as
390 // tokens. These are often punctuation. nonreturnable delimiters
391 // cannot be returned as tokens. These are often whitespace
393 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
394 #if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
395 template <class Char,
396 class Traits = typename std::basic_string<Char>::traits_type >
398 template <class Char,
399 class Traits = std::basic_string<Char>::traits_type >
401 class char_delimiters_separator {
404 typedef std::basic_string<Char,Traits> string_type;
405 string_type returnable_;
406 string_type nonreturnable_;
411 bool is_ret(Char E)const
413 if (returnable_.length())
414 return returnable_.find(E) != string_type::npos;
416 if (no_ispunct_) {return false;}
418 int r = std::ispunct(E);
423 bool is_nonret(Char E)const
425 if (nonreturnable_.length())
426 return nonreturnable_.find(E) != string_type::npos;
428 if (no_isspace_) {return false;}
430 int r = std::isspace(E);
437 explicit char_delimiters_separator(bool return_delims = false,
438 const Char* returnable = 0,
439 const Char* nonreturnable = 0)
440 : returnable_(returnable ? returnable : string_type().c_str()),
441 nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
442 return_delims_(return_delims), no_ispunct_(returnable!=0),
443 no_isspace_(nonreturnable!=0) { }
449 template <typename InputIterator, typename Token>
450 bool operator()(InputIterator& next, InputIterator end,Token& tok) {
453 // skip past all nonreturnable delims
454 // skip past the returnable only if we are not returning delims
455 for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
456 && !return_delims_ ) );++next) { }
462 // if we are to return delims and we are one a returnable one
463 // move past it and stop
464 if (is_ret(*next) && return_delims_) {
469 // append all the non delim characters
470 for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)