src/tex2lyx/Parser.cpp

   1 /**
   2  * \file Parser.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  */
  10
  11 #include <config.h>
  12
  13 #include "Parser.h"
  14
  15 #include "tex2lyx.h"
  16
  17 #include "Encoding.h"
  18 #include "support/lstrings.h"
  19 #include "support/textutils.h"
  20
  21 #include <cstdint>
  22 #include <iostream>
  23
  24 using namespace std;
  25 using namespace lyx::support;
  26
  27 namespace lyx {
  28
  29 namespace {
  30
  31 /*!
  32  * Translate a line ending to '\n'.
  33  * \p c must have catcode catNewline, and it must be the last character read
  34  * from \p is.
  35  */
  36 char_type getNewline(iparserdocstream & is, char_type c)
  37 {
  38         // we have to handle 3 different line endings:
  39         // - UNIX (\n)
  40         // - MAC  (\r)
  41         // - DOS  (\r\n)
  42         if (c == '\r') {
  43                 // MAC or DOS
  44                 char_type wc;
  45                 if (is.get(wc) && wc != '\n') {
  46                         // MAC
  47                         is.putback(wc);
  48                 }
  49                 return '\n';
  50         }
  51         // UNIX
  52         return c;
  53 }
  54
  55 } // namespace
  56
  57 //
  58 // Token
  59 //
  60
  61 ostream & operator<<(ostream & os, Token const & t)
  62 {
  63         if (t.cat() == catComment)
  64                 os << '%' << t.cs() << '\n';
  65         else if (t.cat() == catSpace)
  66                 os << t.cs();
  67         else if (t.cat() == catEscape)
  68                 os << '\\' << t.cs() << ' ';
  69         else if (t.cat() == catLetter)
  70                 os << t.cs();
  71         else if (t.cat() == catNewline)
  72                 os << "[" << t.cs().size() << "\\n," << t.cat() << "]\n";
  73         else
  74                 os << '[' << t.cs() << ',' << t.cat() << ']';
  75         return os;
  76 }
  77
  78
  79 string Token::asInput() const
  80 {
  81         if (cat_ == catComment)
  82                 return '%' + cs_ + '\n';
  83         if (cat_ == catEscape)
  84                 return '\\' + cs_;
  85         return cs_;
  86 }
  87
  88
  89 bool Token::isAlnumASCII() const
  90 {
  91         return cat_ == catLetter ||
  92                (cat_ == catOther && cs_.length() == 1 && isDigitASCII(cs_[0]));
  93 }
  94
  95
  96 #ifdef FILEDEBUG
  97 void debugToken(std::ostream & os, Token const & t, unsigned int flags)
  98 {
  99         char sep = ' ';
 100         os << "t: " << t << " flags: " << flags;
 101         if (flags & FLAG_BRACE_LAST) { os << sep << "BRACE_LAST"; sep = '|'; }
 102         if (flags & FLAG_RIGHT     ) { os << sep << "RIGHT"     ; sep = '|'; }
 103         if (flags & FLAG_END       ) { os << sep << "END"       ; sep = '|'; }
 104         if (flags & FLAG_BRACK_LAST) { os << sep << "BRACK_LAST"; sep = '|'; }
 105         if (flags & FLAG_TEXTMODE  ) { os << sep << "TEXTMODE"  ; sep = '|'; }
 106         if (flags & FLAG_ITEM      ) { os << sep << "ITEM"      ; sep = '|'; }
 107         if (flags & FLAG_LEAVE     ) { os << sep << "LEAVE"     ; sep = '|'; }
 108         if (flags & FLAG_SIMPLE    ) { os << sep << "SIMPLE"    ; sep = '|'; }
 109         if (flags & FLAG_EQUATION  ) { os << sep << "EQUATION"  ; sep = '|'; }
 110         if (flags & FLAG_SIMPLE2   ) { os << sep << "SIMPLE2"   ; sep = '|'; }
 111         if (flags & FLAG_OPTION    ) { os << sep << "OPTION"    ; sep = '|'; }
 112         if (flags & FLAG_BRACED    ) { os << sep << "BRACED"    ; sep = '|'; }
 113         if (flags & FLAG_CELL      ) { os << sep << "CELL"      ; sep = '|'; }
 114         if (flags & FLAG_TABBING   ) { os << sep << "TABBING"   ; sep = '|'; }
 115         os << "\n";
 116 }
 117 #endif
 118
 119
 120 //
 121 // Wrapper
 122 //
 123
 124 void iparserdocstream::setEncoding(std::string const & e)
 125 {
 126         is_ << lyx::setEncoding(e);
 127 }
 128
 129
 130 void iparserdocstream::putback(char_type c)
 131 {
 132         s_ = c + s_;
 133 }
 134
 135
 136 void iparserdocstream::putback(docstring const & s)
 137 {
 138         s_ = s + s_;
 139 }
 140
 141
 142 iparserdocstream & iparserdocstream::get(char_type &c)
 143 {
 144         if (s_.empty())
 145                 is_.get(c);
 146         else {
 147                 //warning_message("unparsed: " + to_utf8(s_));
 148                 c = s_[0];
 149                 s_.erase(0,1);
 150         }
 151         return *this;
 152 }
 153
 154
 155 //
 156 // Parser
 157 //
 158
 159
 160 Parser::Parser(idocstream & is, std::string const & fixedenc)
 161         : lineno_(0), pos_(0), iss_(nullptr), is_(is),
 162           encoding_iconv_(fixedenc.empty() ? "UTF-8" : fixedenc),
 163           theCatcodesType_(NORMAL_CATCODES), curr_cat_(UNDECIDED_CATCODES),
 164           fixed_enc_(!fixedenc.empty())
 165 {
 166         if (fixed_enc_)
 167                 is_.setEncoding(fixedenc);
 168         catInit();
 169 }
 170
 171
 172 Parser::Parser(string const & s)
 173         : lineno_(0), pos_(0),
 174           iss_(new idocstringstream(from_utf8(s))), is_(*iss_),
 175           encoding_iconv_("UTF-8"),
 176           theCatcodesType_(NORMAL_CATCODES), curr_cat_(UNDECIDED_CATCODES),
 177           // An idocstringstream can not change the encoding
 178           fixed_enc_(true)
 179 {
 180         catInit();
 181 }
 182
 183
 184 Parser::~Parser()
 185 {
 186         delete iss_;
 187 }
 188
 189
 190 void Parser::deparse()
 191 {
 192         string s;
 193         for(size_type i = pos_ ; i < tokens_.size() ; ++i) {
 194                 s += tokens_[i].asInput();
 195         }
 196         is_.putback(from_utf8(s));
 197         tokens_.erase(tokens_.begin() + pos_, tokens_.end());
 198         // make sure that next token is read
 199         tokenize_one();
 200 }
 201
 202
 203 bool Parser::setEncoding(std::string const & e, int p)
 204 {
 205         // We may (and need to) use unsafe encodings here: Since the text is
 206         // converted to unicode while reading from is_, we never see text in
 207         // the original encoding of the parser, but operate on utf8 strings
 208         // instead. Therefore, we cannot misparse high bytes as {, } or \\.
 209         Encoding const * const enc = encodings.fromLaTeXName(e, p, true);
 210         if (!enc) {
 211                 warning_message("Unknown encoding " + e + ". Ignoring.");
 212                 return false;
 213         }
 214         return setEncoding(enc->iconvName());
 215 }
 216
 217
 218 void Parser::catInit()
 219 {
 220         if (curr_cat_ == theCatcodesType_)
 221                 return;
 222         curr_cat_ = theCatcodesType_;
 223
 224         fill(theCatcode_, theCatcode_ + 256, catOther);
 225         fill(theCatcode_ + 'a', theCatcode_ + 'z' + 1, catLetter);
 226         fill(theCatcode_ + 'A', theCatcode_ + 'Z' + 1, catLetter);
 227         // This is wrong!
 228         theCatcode_[int('@')]  = catLetter;
 229
 230         if (theCatcodesType_ == NORMAL_CATCODES) {
 231                 theCatcode_[int('\\')] = catEscape;
 232                 theCatcode_[int('{')]  = catBegin;
 233                 theCatcode_[int('}')]  = catEnd;
 234                 theCatcode_[int('$')]  = catMath;
 235                 theCatcode_[int('&')]  = catAlign;
 236                 theCatcode_[int('\n')] = catNewline;
 237                 theCatcode_[int('#')]  = catParameter;
 238                 theCatcode_[int('^')]  = catSuper;
 239                 theCatcode_[int('_')]  = catSub;
 240                 theCatcode_[0x7f]      = catIgnore;
 241                 theCatcode_[int(' ')]  = catSpace;
 242                 theCatcode_[int('\t')] = catSpace;
 243                 theCatcode_[int('\r')] = catNewline;
 244                 theCatcode_[int('~')]  = catActive;
 245                 theCatcode_[int('%')]  = catComment;
 246         }
 247 }
 248
 249 CatCode Parser::catcode(char_type c) const
 250 {
 251         if (c < 256)
 252                 return theCatcode_[(unsigned char)c];
 253         return catOther;
 254 }
 255
 256
 257 void Parser::setCatcode(char c, CatCode cat)
 258 {
 259         theCatcode_[(unsigned char)c] = cat;
 260         deparse();
 261 }
 262
 263
 264 void Parser::setCatcodes(cat_type t)
 265 {
 266         theCatcodesType_ = t;
 267         deparse();
 268 }
 269
 270
 271 bool Parser::setEncoding(std::string const & e)
 272 {
 273         //warning_message("setting encoding to " + e);
 274         encoding_iconv_ = e;
 275         // If the encoding is fixed, we must not change the stream encoding
 276         // (because the whole input uses that encoding, e.g. if it comes from
 277         // the clipboard). We still need to track the original encoding in
 278         // encoding_iconv_, so that the generated output is correct.
 279         if (!fixed_enc_)
 280                 is_.setEncoding(e);
 281         return true;
 282 }
 283
 284
 285 void Parser::push_back(Token const & t)
 286 {
 287         tokens_.push_back(t);
 288 }
 289
 290
 291 // We return a copy here because the tokens_ vector may get reallocated
 292 Token const Parser::prev_token() const
 293 {
 294         static const Token dummy;
 295         return pos_ > 1 ? tokens_[pos_ - 2] : dummy;
 296 }
 297
 298
 299 // We return a copy here because the tokens_ vector may get reallocated
 300 Token const Parser::curr_token() const
 301 {
 302         static const Token dummy;
 303         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 304 }
 305
 306
 307 // We return a copy here because the tokens_ vector may get reallocated
 308 Token const Parser::next_token()
 309 {
 310         static const Token dummy;
 311         if (!good())
 312                 return dummy;
 313         if (pos_ >= tokens_.size())
 314                 tokenize_one();
 315         return pos_ < tokens_.size() ? tokens_[pos_] : dummy;
 316 }
 317
 318
 319 // We return a copy here because the tokens_ vector may get reallocated
 320 Token const Parser::next_next_token()
 321 {
 322         static const Token dummy;
 323         if (!good())
 324                 return dummy;
 325         // If tokenize_one() has not been called after the last get_token() we
 326         // need to tokenize two more tokens.
 327         if (pos_ >= tokens_.size())
 328                 tokenize_one();
 329         if (pos_ + 1 >= tokens_.size())
 330                 tokenize_one();
 331         return pos_ + 1 < tokens_.size() ? tokens_[pos_ + 1] : dummy;
 332 }
 333
 334
 335 // We return a copy here because the tokens_ vector may get reallocated
 336 Token const Parser::get_token()
 337 {
 338         static const Token dummy;
 339         if (!good())
 340                 return dummy;
 341         if (pos_ >= tokens_.size()) {
 342                 tokenize_one();
 343                 if (pos_ >= tokens_.size())
 344                         return dummy;
 345         }
 346         // warning_message("looking at token " + tokens_[pos_]
 347         //      + " pos: " + pos_ <<);
 348         return tokens_[pos_++];
 349 }
 350
 351
 352 bool Parser::isParagraph()
 353 {
 354         // A new paragraph in TeX is started
 355         // - either by a newline, following any amount of whitespace
 356         //   characters (including zero), and another newline
 357         // - or the token \par
 358         if (curr_token().cat() == catNewline &&
 359             (curr_token().cs().size() > 1 ||
 360              (next_token().cat() == catSpace &&
 361               next_next_token().cat() == catNewline)))
 362                 return true;
 363         if (curr_token().cat() == catEscape && curr_token().cs() == "par")
 364                 return true;
 365         return false;
 366 }
 367
 368
 369 bool Parser::skip_spaces(bool skip_comments)
 370 {
 371         // We just silently return if we have no more tokens.
 372         // skip_spaces() should be callable at any time,
 373         // the caller must check p::good() anyway.
 374         bool skipped = false;
 375         while (good()) {
 376                 get_token();
 377                 if (isParagraph()) {
 378                         putback();
 379                         break;
 380                 }
 381                 if (curr_token().cat() == catSpace ||
 382                     curr_token().cat() == catNewline) {
 383                         skipped = true;
 384                         continue;
 385                 }
 386                 if ((curr_token().cat() == catComment && curr_token().cs().empty()))
 387                         continue;
 388                 if (skip_comments && curr_token().cat() == catComment) {
 389                         // If positions_ is not empty we are doing some kind
 390                         // of look ahead
 391                         if (!positions_.empty())
 392                                 warning_message("Ignoring comment: " + curr_token().asInput());
 393                 } else {
 394                         putback();
 395                         break;
 396                 }
 397         }
 398         return skipped;
 399 }
 400
 401
 402 void Parser::unskip_spaces(bool skip_comments)
 403 {
 404         while (pos_ > 0) {
 405                 if ( curr_token().cat() == catSpace ||
 406                     (curr_token().cat() == catNewline && curr_token().cs().size() == 1))
 407                         putback();
 408                 else if (skip_comments && curr_token().cat() == catComment) {
 409                         // TODO: Get rid of this
 410                         // If positions_ is not empty we are doing some kind
 411                         // of look ahead
 412                         if (!positions_.empty())
 413                                 warning_message("Unignoring comment: " + curr_token().asInput());
 414                         putback();
 415                 }
 416                 else
 417                         break;
 418         }
 419 }
 420
 421
 422 void Parser::putback()
 423 {
 424         --pos_;
 425 }
 426
 427
 428 void Parser::pushPosition()
 429 {
 430         positions_.push_back(pos_);
 431 }
 432
 433
 434 void Parser::popPosition()
 435 {
 436         pos_ = positions_.back();
 437         positions_.pop_back();
 438         deparse();
 439 }
 440
 441
 442 void Parser::dropPosition()
 443 {
 444         positions_.pop_back();
 445 }
 446
 447
 448 bool Parser::good() const
 449 {
 450         if (pos_ < tokens_.size())
 451                 return true;
 452         if (!is_.good())
 453                 return false;
 454         return is_.peek() != idocstream::traits_type::eof();
 455 }
 456
 457
 458 bool Parser::hasOpt(string const & l)
 459 {
 460         // An optional argument can occur in any of the following forms:
 461         // - \foo[bar]
 462         // - \foo [bar]
 463         // - \foo
 464         //   [bar]
 465         // - \foo %comment
 466         //   [bar]
 467
 468         // remember current position
 469         unsigned int oldpos = pos_;
 470         // skip spaces and comments
 471         while (good()) {
 472                 get_token();
 473                 if (isParagraph()) {
 474                         putback();
 475                         break;
 476                 }
 477                 if (curr_token().cat() == catSpace ||
 478                     curr_token().cat() == catNewline ||
 479                     curr_token().cat() == catComment)
 480                         continue;
 481                 putback();
 482                 break;
 483         }
 484         bool const retval = (next_token().asInput() == l);
 485         pos_ = oldpos;
 486         return retval;
 487 }
 488
 489
 490 bool Parser::hasIdxMacros(string const & c, string const & e)
 491 {
 492         // Check for index entry separator (! or @),
 493         // consider escaping via "
 494         // \p e marks a terminating delimiter¸
 495
 496         // remember current position
 497         unsigned int oldpos = pos_;
 498         // skip spaces and comments
 499         bool retval = false;
 500         while (good()) {
 501                 get_token();
 502                 if (isParagraph()) {
 503                         putback();
 504                         break;
 505                 }
 506                 if (curr_token().cat() == catEnd)
 507                         break;
 508                 if (!e.empty() && curr_token().asInput() == e
 509                     && prev_token().asInput() != "\"")
 510                         break;
 511                 if (curr_token().asInput() == c
 512                     && prev_token().asInput() != "\"") {
 513                         retval = true;
 514                         break;
 515                 }
 516                 continue;
 517         }
 518         pos_ = oldpos;
 519         return retval;
 520 }
 521
 522
 523 Parser::Arg Parser::getFullArg(char left, char right, bool allow_escaping, char e)
 524 {
 525         skip_spaces(true);
 526
 527         // This is needed if a partial file ends with a command without arguments,
 528         // e. g. \medskip
 529         if (! good())
 530                 return make_pair(false, string());
 531
 532         int group_level = (left == '{') ? 1 : 0;
 533         string result;
 534         Token t = get_token();
 535
 536         if (left != char()
 537             && (t.cat() == catComment || t.cat() == catEscape
 538                 || t.character() != left)) {
 539                 putback();
 540                 return make_pair(false, string());
 541         } else {
 542                 while (good()) {
 543                         t = get_token();
 544                         // honor grouping
 545                         if (t.cat() == catBegin) {
 546                                 ++group_level;
 547                                 if (left != '{')
 548                                         continue;
 549                         }
 550                         if (group_level > 0 && t.cat() == catEnd) {
 551                                 --group_level;
 552                                 if (left != '{')
 553                                         continue;
 554                         }
 555                         // Ignore comments
 556                         if (t.cat() == catComment) {
 557                                 if (!t.cs().empty())
 558                                         warning_message("Ignoring comment: " + t.asInput());
 559                                 continue;
 560                         }
 561                         if (allow_escaping) {
 562                                 if (t.cat() != catEscape && t.character() == right
 563                                     && group_level == 0)
 564                                         break;
 565                         } else if (e != char()) {
 566                                 if (prev_token().character() != e && t.character() == right
 567                                     && group_level == 0)
 568                                         break;
 569                         } else {
 570                                 if (t.character() == right) {
 571                                         if (t.cat() == catEscape)
 572                                                 result += '\\';
 573                                         if (group_level == 0)
 574                                                 break;
 575                                 }
 576                         }
 577                         result += t.asInput();
 578                 }
 579         }
 580         return make_pair(true, result);
 581 }
 582
 583
 584 string Parser::getArg(char left, char right, bool allow_escaping, char e)
 585 {
 586         return getFullArg(left, right, allow_escaping, e).second;
 587 }
 588
 589
 590 string Parser::getFullOpt(bool keepws, char left, char right)
 591 {
 592         Arg arg = getFullArg(left, right);
 593         if (arg.first)
 594                 return left + arg.second + right;
 595         if (keepws)
 596                 unskip_spaces(true);
 597         return string();
 598 }
 599
 600
 601 string Parser::getOpt(bool keepws)
 602 {
 603         string const res = getArg('[', ']');
 604         if (res.empty()) {
 605                 if (keepws)
 606                         unskip_spaces(true);
 607                 return string();
 608         }
 609         return '[' + res + ']';
 610 }
 611
 612
 613 string Parser::getFullParentheseArg()
 614 {
 615         Arg arg = getFullArg('(', ')');
 616         if (arg.first)
 617                 return '(' + arg.second + ')';
 618         return string();
 619 }
 620
 621
 622 bool Parser::hasListPreamble(string const & itemcmd)
 623 {
 624         // remember current position
 625         unsigned int oldpos = pos_;
 626         // jump over arguments
 627         if (hasOpt())
 628                 getOpt();
 629         if (hasOpt("{"))
 630                 getArg('{', '}');
 631         // and swallow spaces and comments
 632         skip_spaces(true);
 633         // we have a list preamble if the next thing
 634         // that follows is not the \item command
 635         bool res =  next_token().cs() != itemcmd;
 636         // back to orig position
 637         pos_ = oldpos;
 638         return res;
 639 }
 640
 641
 642 string const Parser::ertEnvironment(string const & name)
 643 {
 644         if (!good())
 645                 return string();
 646
 647         ostringstream os;
 648         for (Token t = get_token(); good(); t = get_token()) {
 649                 if (t.cat() == catBegin) {
 650                         putback();
 651                         os << '{' << verbatim_item() << '}';
 652                 } else if (t.asInput() == "\\begin") {
 653                         string const env = getArg('{', '}');
 654                         os << "\\begin{" << env << '}'
 655                            << ertEnvironment(env)
 656                            << "\\end{" << env << '}';
 657                 } else if (t.asInput() == "\\end") {
 658                         string const end = getArg('{', '}');
 659                         if (end != name)
 660                                 warning_message("\\end{" + end
 661                                                 + "} does not match \\begin{"
 662                                                 + name + "}.");
 663                         return os.str();
 664                 } else
 665                         os << t.asInput();
 666         }
 667         warning_message("unexpected end of input");
 668         return os.str();
 669 }
 670
 671
 672 string const Parser::plainEnvironment(string const & name)
 673 {
 674         if (!good())
 675                 return string();
 676
 677         ostringstream os;
 678         for (Token t = get_token(); good(); t = get_token()) {
 679                 if (t.asInput() == "\\end") {
 680                         string const end = getArg('{', '}');
 681                         if (end == name)
 682                                 return os.str();
 683                         else
 684                                 os << "\\end{" << end << '}';
 685                 } else
 686                         os << t.asInput();
 687         }
 688         warning_message("unexpected end of input");
 689         return os.str();
 690 }
 691
 692
 693 string const Parser::plainCommand(char left, char right, string const & name)
 694 {
 695         if (!good())
 696                 return string();
 697         // check if first token is really the start character
 698         Token tok = get_token();
 699         if (tok.character() != left) {
 700                 warning_message("first character does not match start character of command \\" + name);
 701                 return string();
 702         }
 703         ostringstream os;
 704         for (Token t = get_token(); good(); t = get_token()) {
 705                 if (t.character() == right) {
 706                         return os.str();
 707                 } else
 708                         os << t.asInput();
 709         }
 710         warning_message("unexpected end of input");
 711         return os.str();
 712 }
 713
 714
 715 string const Parser::getCommandLatexParam()
 716 {
 717         if (!good())
 718                 return string();
 719         string res;
 720         size_t offset = 0;
 721         while (true) {
 722                 if (pos_ + offset >= tokens_.size())
 723                         tokenize_one();
 724                 if (pos_ + offset >= tokens_.size())
 725                         break;
 726                 Token t = tokens_[pos_ + offset];
 727                 if (t.cat() == catBegin)
 728                         break;
 729                 res += t.asInput();
 730                 ++offset;
 731         }
 732         return res;
 733 }
 734
 735
 736 Parser::Arg Parser::verbatimStuff(string const & end_string, bool const allow_linebreak)
 737 {
 738         if (!good())
 739                 return Arg(false, string());
 740
 741         pushPosition();
 742         ostringstream oss;
 743         size_t match_index = 0;
 744         setCatcodes(VERBATIM_CATCODES);
 745         for (Token t = get_token(); good(); t = get_token()) {
 746                 // FIXME t.asInput() might be longer than we need ?
 747                 if (t.asInput() == end_string.substr(match_index,
 748                                                      t.asInput().length())) {
 749                         match_index += t.asInput().length();
 750                         if (match_index >= end_string.length())
 751                                 break;
 752                 } else {
 753                         if (!allow_linebreak && t.asInput() == "\n") {
 754                                 warning_message("unexpected end of input");
 755                                 popPosition();
 756                                 setCatcodes(NORMAL_CATCODES);
 757                                 return Arg(false, string());
 758                         }
 759                         if (match_index) {
 760                                 oss << end_string.substr(0, match_index)
 761                                     << t.asInput();
 762                                 match_index = 0;
 763                         } else
 764                                 oss << t.asInput();
 765                 }
 766         }
 767
 768         if (!good()) {
 769                 warning_message("unexpected end of input");
 770                 popPosition();
 771                 setCatcodes(NORMAL_CATCODES);
 772                 return Arg(false, string());
 773         }
 774         setCatcodes(NORMAL_CATCODES);
 775         dropPosition();
 776         return Arg(true, oss.str());
 777 }
 778
 779
 780 string const Parser::verbatimEnvironment(string const & name)
 781 {
 782         //FIXME: do something if endstring is not found
 783         string s = verbatimStuff("\\end{" + name + "}").second;
 784         // ignore one newline at beginning or end of string
 785         if (prefixIs(s, "\n"))
 786                 s.erase(0,1);
 787         if (suffixIs(s, "\n"))
 788                 s.erase(s.length() - 1,1);
 789         return s;
 790 }
 791
 792
 793 string Parser::verbatimOption()
 794 {
 795         string res;
 796         if (next_token().character() == '[') {
 797                 Token t = get_token();
 798                 for (t = get_token(); t.character() != ']' && good(); t = get_token()) {
 799                         if (t.cat() == catBegin) {
 800                                 putback();
 801                                 res += '{' + verbatim_item() + '}';
 802                         } else
 803                                 res += t.asInput();
 804                 }
 805         }
 806         return res;
 807 }
 808
 809
 810 string Parser::verbatim_item()
 811 {
 812         if (!good())
 813                 error("stream bad");
 814         skip_spaces();
 815         if (next_token().cat() == catBegin) {
 816                 Token t = get_token(); // skip brace
 817                 string res;
 818                 for (t = get_token(); t.cat() != catEnd && good(); t = get_token()) {
 819                         if (t.cat() == catBegin) {
 820                                 putback();
 821                                 res += '{' + verbatim_item() + '}';
 822                         }
 823                         else
 824                                 res += t.asInput();
 825                 }
 826                 return res;
 827         }
 828         return get_token().asInput();
 829 }
 830
 831
 832 void Parser::tokenize_one()
 833 {
 834         catInit();
 835         char_type c;
 836         if (!is_.get(c))
 837                 return;
 838
 839         switch (catcode(c)) {
 840         case catSpace: {
 841                 docstring s(1, c);
 842                 while (is_.get(c) && catcode(c) == catSpace)
 843                         s += c;
 844                 if (catcode(c) != catSpace)
 845                         is_.putback(c);
 846                 push_back(Token(s, catSpace));
 847                 break;
 848         }
 849
 850         case catNewline: {
 851                 ++lineno_;
 852                 docstring s(1, getNewline(is_, c));
 853                 while (is_.get(c) && catcode(c) == catNewline) {
 854                         ++lineno_;
 855                         s += getNewline(is_, c);
 856                 }
 857                 if (catcode(c) != catNewline)
 858                         is_.putback(c);
 859                 push_back(Token(s, catNewline));
 860                 break;
 861         }
 862
 863         case catComment: {
 864                 // We don't treat "%\n" combinations here specially because
 865                 // we want to preserve them in the preamble
 866                 docstring s;
 867                 while (is_.get(c) && catcode(c) != catNewline)
 868                         s += c;
 869                 // handle possible DOS line ending
 870                 if (catcode(c) == catNewline)
 871                         c = getNewline(is_, c);
 872                 // Note: The '%' at the beginning and the '\n' at the end
 873                 // of the comment are not stored.
 874                 ++lineno_;
 875                 push_back(Token(s, catComment));
 876                 break;
 877         }
 878
 879         case catEscape: {
 880                 is_.get(c);
 881                 if (!is_) {
 882                         error("unexpected end of input");
 883                 } else {
 884                         docstring s(1, c);
 885                         if (catcode(c) == catLetter) {
 886                                 // collect letters
 887                                 while (is_.get(c) && catcode(c) == catLetter)
 888                                         s += c;
 889                                 if (catcode(c) != catLetter)
 890                                         is_.putback(c);
 891                         }
 892                         push_back(Token(s, catEscape));
 893                 }
 894                 break;
 895         }
 896
 897         case catIgnore: {
 898                 warning_message("ignoring a char: " + std::to_string(static_cast<uint32_t>(c)));
 899                 break;
 900         }
 901
 902         default:
 903                 push_back(Token(docstring(1, c), catcode(c)));
 904         }
 905         //warning_message(tokens_.back());
 906 }
 907
 908
 909 void Parser::dump() const
 910 {
 911         cerr << "\nTokens: ";
 912         for (unsigned i = 0; i < tokens_.size(); ++i) {
 913                 if (i == pos_)
 914                         cerr << " <#> ";
 915                 cerr << tokens_[i];
 916         }
 917         cerr << " pos: " << pos_ << "\n";
 918 }
 919
 920
 921 void Parser::error(string const & msg) const
 922 {
 923         cerr << "Line ~" << lineno_ << ":  parse error: " << msg << endl;
 924         dump();
 925         //exit(1);
 926 }
 927
 928
 929 void Parser::reset()
 930 {
 931         pos_ = 0;
 932 }
 933
 934
 935 } // namespace lyx