1 // Boost token_functions.hpp ------------------------------------------------//
3 // Copyright John R. Bandela 2001.
5 // Permission to copy, use, modify, sell and distribute this software
6 // is granted provided this copyright notice appears in all
7 // copies. This software is provided "as is" without express or
8 // implied warranty, and with no claim as to its suitability for any
11 // See http://www.boost.org for updates, documentation, and revision history.
15 // 20 Feb 2002 John Maddock
16 // Removed using namespace std declarations and added
17 // workaround for BOOST_NO_STDC_NAMESPACE (the library
18 // can be safely mixed with regex).
19 // 06 Feb 2002 Jeremy Siek
20 // Added char_separator.
21 // 02 Feb 2002 Jeremy Siek
22 // Removed tabs and a little cleanup.
25 #ifndef BOOST_TOKEN_FUNCTIONS_JRB051801_HPP_
26 #define BOOST_TOKEN_FUNCTIONS_JRB051801_HPP_
35 // the following must not be macros if we are to prefix them
36 // with std:: (they shouldn't be macros anyway...)
45 // fix namespace problems:
47 #ifdef BOOST_NO_STDC_NAMESPACE
56 //===========================================================================
57 // The escaped_list_separator class. Which is a model of TokenizerFunction
58 // An escaped list is a super-set of what is commonly known as a comma
59 // separated value (csv) list.It is separated into fields by a comma or
60 // other character. If the delimiting character is inside quotes, then it is
61 // counted as a regular character.To allow for embedded quotes in a field,
62 // there can be escape sequences using the \ much like C.
63 // The role of the comma, the quotation mark, and the escape
64 // character (backslash \), can be assigned to other characters.
66 struct escaped_list_error : public std::runtime_error{
67 escaped_list_error(const std::string& what):std::runtime_error(what) { }
71 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
72 // MSVC does not like the following typename
75 class Traits = typename std::basic_string<Char>::traits_type >
78 class Traits = std::basic_string<Char>::traits_type >
80 class escaped_list_separator {
83 typedef std::basic_string<Char,Traits> string_type;
86 char_eq(Char e):e_(e) { }
87 bool operator()(Char c) {
88 return Traits::eq(e_,c);
96 bool is_escape(Char e) {
98 return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
102 return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
104 bool is_quote(Char e) {
106 return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
108 template <typename iterator, typename Token>
109 void do_escape(iterator& next,iterator end,Token& tok) {
111 throw escaped_list_error(std::string("cannot end with escape"));
112 if (Traits::eq(*next,'n')) {
116 else if (is_quote(*next)) {
120 else if (is_c(*next)) {
124 else if (is_escape(*next)) {
129 throw escaped_list_error(std::string("unknown escape sequence"));
134 explicit escaped_list_separator(Char e = '\\',
135 Char c = ',',Char q = '\"')
136 : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
138 escaped_list_separator(string_type e, string_type c, string_type q)
139 : escape_(e), c_(c), quote_(q), last_(false) { }
141 void reset() {last_=false;}
143 template <typename InputIterator, typename Token>
144 bool operator()(InputIterator& next,InputIterator end,Token& tok) {
145 bool bInQuote = false;
157 for (;next != end;++next) {
158 if (is_escape(*next)) {
159 do_escape(next,end,tok);
161 else if (is_c(*next)) {
163 // If we are not in quote, then we are done
165 // The last character was a c, that means there is
166 // 1 more blank field
172 else if (is_quote(*next)) {
184 //===========================================================================
185 // The offset_separator class, which is a model of TokenizerFunction.
186 // Offset breaks a string into tokens based on a range of offsets
188 class offset_separator {
191 std::vector<int> offsets_;
192 unsigned int current_offset_;
194 bool return_partial_last_;
197 template <typename Iter>
198 offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
199 bool return_partial_last = true)
200 : offsets_(begin,end), current_offset_(0),
201 wrap_offsets_(wrap_offsets),
202 return_partial_last_(return_partial_last) { }
205 : offsets_(1,1), current_offset_(),
206 wrap_offsets_(true), return_partial_last_(true) { }
212 template <typename InputIterator, typename Token>
213 bool operator()(InputIterator& next, InputIterator end, Token& tok)
215 assert(!offsets_.empty());
222 if (current_offset_ == offsets_.size())
228 int c = offsets_[current_offset_];
231 if (next == end)break;
235 if (!return_partial_last_)
245 //===========================================================================
246 // The char_separator class breaks a sequence of characters into
247 // tokens based on the character delimiters (very much like bad old
248 // strtok). A delimiter character can either be kept or dropped. A
249 // kept delimiter shows up as an output token, whereas a dropped
250 // delimiter does not.
252 // This class replaces the char_delimiters_separator class. The
253 // constructor for the char_delimiters_separator class was too
254 // confusing and needed to be deprecated. However, because of the
255 // default arguments to the constructor, adding the new constructor
256 // would cause ambiguity, so instead I deprecated the whole class.
257 // The implementation of the class was also simplified considerably.
259 enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
261 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
263 template <typename Char,
264 typename Traits = typename std::basic_string<Char>::traits_type >
266 template <typename Char,
267 typename Traits = std::basic_string<Char>::traits_type >
271 typedef std::basic_string<Char,Traits> string_type;
274 char_separator(const Char* dropped_delims,
275 const Char* kept_delims = 0,
276 empty_token_policy empty_tokens = drop_empty_tokens)
277 : m_dropped_delims(dropped_delims),
278 m_use_ispunct(false),
279 m_use_isspace(false),
280 m_empty_tokens(empty_tokens),
283 // Borland workaround
285 m_kept_delims = kept_delims;
288 // use ispunct() for kept delimiters and isspace for dropped.
291 : m_use_ispunct(true),
293 m_empty_tokens(drop_empty_tokens) { }
297 template <typename InputIterator, typename Token>
298 bool operator()(InputIterator& next, InputIterator end, Token& tok)
302 // skip past all dropped_delims
303 if (m_empty_tokens == drop_empty_tokens)
304 for (; next != end && is_dropped(*next); ++next)
307 if (m_empty_tokens == drop_empty_tokens) {
312 // if we are on a kept_delims move past it and stop
313 if (is_kept(*next)) {
317 // append all the non delim characters
318 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
321 else { // m_empty_tokens == keep_empty_tokens
323 // Handle empty token at the end
325 if (m_output_done == false) {
326 m_output_done = true;
331 if (is_kept(*next)) {
332 if (m_output_done == false)
333 m_output_done = true;
337 m_output_done = false;
340 else if (m_output_done == false && is_dropped(*next)) {
341 m_output_done = true;
344 if (is_dropped(*next))
346 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
348 m_output_done = true;
355 string_type m_kept_delims;
356 string_type m_dropped_delims;
359 empty_token_policy m_empty_tokens;
362 bool is_kept(Char E) const
364 if (m_kept_delims.length())
365 return m_kept_delims.find(E) != string_type::npos;
366 else if (m_use_ispunct) {
367 return std::ispunct(E) != 0;
371 bool is_dropped(Char E) const
373 if (m_dropped_delims.length())
374 return m_dropped_delims.find(E) != string_type::npos;
375 else if (m_use_isspace) {
376 return std::isspace(E) != 0;
382 //===========================================================================
383 // The following class is DEPRECATED, use class char_separators instead.
385 // The char_delimiters_separator class, which is a model of
386 // TokenizerFunction. char_delimiters_separator breaks a string
387 // into tokens based on character delimiters. There are 2 types of
388 // delimiters. returnable delimiters can be returned as
389 // tokens. These are often punctuation. nonreturnable delimiters
390 // cannot be returned as tokens. These are often whitespace
392 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
394 template <class Char,
395 class Traits = typename std::basic_string<Char>::traits_type >
397 template <class Char,
398 class Traits = std::basic_string<Char>::traits_type >
400 class char_delimiters_separator {
403 typedef std::basic_string<Char,Traits> string_type;
404 string_type returnable_;
405 string_type nonreturnable_;
410 bool is_ret(Char E)const
412 if (returnable_.length())
413 return returnable_.find(E) != string_type::npos;
415 if (no_ispunct_) {return false;}
417 int r = std::ispunct(E);
422 bool is_nonret(Char E)const
424 if (nonreturnable_.length())
425 return nonreturnable_.find(E) != string_type::npos;
427 if (no_isspace_) {return false;}
429 int r = std::isspace(E);
436 explicit char_delimiters_separator(bool return_delims = false,
437 const Char* returnable = 0,
438 const Char* nonreturnable = 0)
439 : returnable_(returnable ? returnable : string_type().c_str()),
440 nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
441 return_delims_(return_delims), no_ispunct_(returnable!=0),
442 no_isspace_(nonreturnable!=0) { }
448 template <typename InputIterator, typename Token>
449 bool operator()(InputIterator& next, InputIterator end,Token& tok) {
452 // skip past all nonreturnable delims
453 // skip past the returnable only if we are not returning delims
454 for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
455 && !return_delims_ ) );++next) { }
461 // if we are to return delims and we are one a returnable one
462 // move past it and stop
463 if (is_ret(*next) && return_delims_) {
468 // append all the non delim characters
469 for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)