+ catInit();
+ char_type c;
+ if (!is_.get(c))
+ return;
+
+ switch (catcode(c)) {
+ case catSpace: {
+ docstring s(1, c);
+ while (is_.get(c) && catcode(c) == catSpace)
+ s += c;
+ if (catcode(c) != catSpace)
+ is_.putback(c);
+ push_back(Token(s, catSpace));
+ break;
+ }
+
+ case catNewline: {
+ ++lineno_;
+ docstring s(1, getNewline(is_, c));
+ while (is_.get(c) && catcode(c) == catNewline) {
+ ++lineno_;
+ s += getNewline(is_, c);
+ }
+ if (catcode(c) != catNewline)
+ is_.putback(c);
+ push_back(Token(s, catNewline));
+ break;
+ }
+
+ case catComment: {
+ // We don't treat "%\n" combinations here specially because
+ // we want to preserve them in the preamble
+ docstring s;
+ while (is_.get(c) && catcode(c) != catNewline)
+ s += c;
+ // handle possible DOS line ending
+ if (catcode(c) == catNewline)
+ c = getNewline(is_, c);
+ // Note: The '%' at the beginning and the '\n' at the end
+ // of the comment are not stored.
+ ++lineno_;
+ push_back(Token(s, catComment));
+ break;
+ }
+
+ case catEscape: {
+ is_.get(c);
+ if (!is_) {
+ error("unexpected end of input");
+ } else {
+ docstring s(1, c);
+ if (catcode(c) == catLetter) {
+ // collect letters
+ while (is_.get(c) && catcode(c) == catLetter)
+ s += c;
+ if (catcode(c) != catLetter)
+ is_.putback(c);
+ }
+ push_back(Token(s, catEscape));
+ }
+ break;
+ }
+
+ case catIgnore: {
+ cerr << "ignoring a char: " << c << "\n";
+ break;
+ }
+
+ default:
+ push_back(Token(docstring(1, c), catcode(c)));
+ }
+ //cerr << tokens_.back();
+}
+
+
+void Parser::dump() const
+{
+ cerr << "\nTokens: ";
+ for (unsigned i = 0; i < tokens_.size(); ++i) {
+ if (i == pos_)
+ cerr << " <#> ";
+ cerr << tokens_[i];
+ }
+ cerr << " pos: " << pos_ << "\n";