src/tex2lyx/Parser.cpp

   1 /**
   2  * \file Parser.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  */
  10
  11 #include <config.h>
  12
  13 #include "Parser.h"
  14
  15 #include <iostream>
  16 #include <sstream>
  17
  18 using namespace std;
  19
  20 namespace lyx {
  21
  22 namespace {
  23
  24 CatCode theCatcode[256];
  25
  26 void catInit()
  27 {
  28         fill(theCatcode, theCatcode + 256, catOther);
  29         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
  30         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
  31
  32         theCatcode[int('\\')] = catEscape;
  33         theCatcode[int('{')]  = catBegin;
  34         theCatcode[int('}')]  = catEnd;
  35         theCatcode[int('$')]  = catMath;
  36         theCatcode[int('&')]  = catAlign;
  37         theCatcode[int('\n')] = catNewline;
  38         theCatcode[int('#')]  = catParameter;
  39         theCatcode[int('^')]  = catSuper;
  40         theCatcode[int('_')]  = catSub;
  41         theCatcode[0x7f]      = catIgnore;
  42         theCatcode[int(' ')]  = catSpace;
  43         theCatcode[int('\t')] = catSpace;
  44         theCatcode[int('\r')] = catNewline;
  45         theCatcode[int('~')]  = catActive;
  46         theCatcode[int('%')]  = catComment;
  47
  48         // This is wrong!
  49         theCatcode[int('@')]  = catLetter;
  50 }
  51
  52
  53 /*!
  54  * Translate a line ending to '\n'.
  55  * \p c must have catcode catNewline, and it must be the last character read
  56  * from \p is.
  57  */
  58 char getNewline(istream & is, char c)
  59 {
  60         // we have to handle 3 different line endings:
  61         // - UNIX (\n)
  62         // - MAC  (\r)
  63         // - DOS  (\r\n)
  64         if (c == '\r') {
  65                 // MAC or DOS
  66                 if (is.get(c) && c != '\n') {
  67                         // MAC
  68                         is.putback(c);
  69                 }
  70                 return '\n';
  71         }
  72         // UNIX
  73         return c;
  74 }
  75
  76 }
  77
  78
  79 //
  80 // catcodes
  81 //
  82
  83 CatCode catcode(unsigned char c)
  84 {
  85         return theCatcode[c];
  86 }
  87
  88
  89
  90 //
  91 // Token
  92 //
  93
  94 ostream & operator<<(ostream & os, Token const & t)
  95 {
  96         if (t.cat() == catComment)
  97                 os << '%' << t.cs() << '\n';
  98         else if (t.cat() == catSpace)
  99                 os << t.cs();
 100         else if (t.cat() == catEscape)
 101                 os << '\\' << t.cs() << ' ';
 102         else if (t.cat() == catLetter)
 103                 os << t.character();
 104         else if (t.cat() == catNewline)
 105                 os << "[" << t.cs().size() << "\\n," << t.cat() << "]\n";
 106         else
 107                 os << '[' << t.character() << ',' << t.cat() << ']';
 108         return os;
 109 }
 110
 111
 112 string Token::asString() const
 113 {
 114         return cs_.size() ? cs_ : string(1, char_);
 115 }
 116
 117
 118 string Token::asInput() const
 119 {
 120         if (cat_ == catComment)
 121                 return '%' + cs_ + '\n';
 122         if (cat_ == catSpace || cat_ == catNewline)
 123                 return cs_;
 124         return char_ ? string(1, char_) : '\\' + cs_;
 125 }
 126
 127
 128 //
 129 // Parser
 130 //
 131
 132
 133 Parser::Parser(istream & is)
 134         : lineno_(0), pos_(0)
 135 {
 136         tokenize(is);
 137 }
 138
 139
 140 Parser::Parser(string const & s)
 141         : lineno_(0), pos_(0)
 142 {
 143         istringstream is(s);
 144         tokenize(is);
 145 }
 146
 147
 148 void Parser::push_back(Token const & t)
 149 {
 150         tokens_.push_back(t);
 151 }
 152
 153
 154 void Parser::pop_back()
 155 {
 156         tokens_.pop_back();
 157 }
 158
 159
 160 Token const & Parser::prev_token() const
 161 {
 162         static const Token dummy;
 163         return pos_ > 1 ? tokens_[pos_ - 2] : dummy;
 164 }
 165
 166
 167 Token const & Parser::curr_token() const
 168 {
 169         static const Token dummy;
 170         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 171 }
 172
 173
 174 Token const & Parser::next_token() const
 175 {
 176         static const Token dummy;
 177         return good() ? tokens_[pos_] : dummy;
 178 }
 179
 180
 181 Token const & Parser::get_token()
 182 {
 183         static const Token dummy;
 184         //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
 185         return good() ? tokens_[pos_++] : dummy;
 186 }
 187
 188
 189 bool Parser::isParagraph() const
 190 {
 191         // A new paragraph in TeX ist started
 192         // - either by a newline, following any amount of whitespace
 193         //   characters (including zero), and another newline
 194         // - or the token \par
 195         if (curr_token().cat() == catNewline &&
 196             (curr_token().cs().size() > 1 ||
 197              (next_token().cat() == catSpace &&
 198               pos_ < tokens_.size() - 1 &&
 199               tokens_[pos_ + 1].cat() == catNewline)))
 200                 return true;
 201         if (curr_token().cat() == catEscape && curr_token().cs() == "par")
 202                 return true;
 203         return false;
 204 }
 205
 206
 207 void Parser::skip_spaces(bool skip_comments)
 208 {
 209         // We just silently return if we have no more tokens.
 210         // skip_spaces() should be callable at any time,
 211         // the caller must check p::good() anyway.
 212         while (good()) {
 213                 get_token();
 214                 if (isParagraph()) {
 215                         putback();
 216                         break;
 217                 }
 218                 if ( curr_token().cat() == catSpace ||
 219                      curr_token().cat() == catNewline ||
 220                     (curr_token().cat() == catComment && curr_token().cs().empty()))
 221                         continue;
 222                 if (skip_comments && curr_token().cat() == catComment)
 223                         cerr << "  Ignoring comment: " << curr_token().asInput();
 224                 else {
 225                         putback();
 226                         break;
 227                 }
 228         }
 229 }
 230
 231
 232 void Parser::unskip_spaces(bool skip_comments)
 233 {
 234         while (pos_ > 0) {
 235                 if ( curr_token().cat() == catSpace ||
 236                     (curr_token().cat() == catNewline && curr_token().cs().size() == 1))
 237                         putback();
 238                 else if (skip_comments && curr_token().cat() == catComment) {
 239                         // TODO: Get rid of this
 240                         cerr << "Unignoring comment: " << curr_token().asInput();
 241                         putback();
 242                 }
 243                 else
 244                         break;
 245         }
 246 }
 247
 248
 249 void Parser::putback()
 250 {
 251         --pos_;
 252 }
 253
 254
 255 bool Parser::good() const
 256 {
 257         return pos_ < tokens_.size();
 258 }
 259
 260
 261 char Parser::getChar()
 262 {
 263         if (!good())
 264                 error("The input stream is not well...");
 265         return tokens_[pos_++].character();
 266 }
 267
 268
 269 Parser::Arg Parser::getFullArg(char left, char right)
 270 {
 271         skip_spaces(true);
 272
 273         // This is needed if a partial file ends with a command without arguments,
 274         // e. g. \medskip
 275         if (! good())
 276                 return make_pair(false, string());
 277
 278         string result;
 279         char c = getChar();
 280
 281         if (c != left) {
 282                 putback();
 283                 return make_pair(false, string());
 284         } else
 285                 while ((c = getChar()) != right && good()) {
 286                         // Ignore comments
 287                         if (curr_token().cat() == catComment) {
 288                                 if (!curr_token().cs().empty())
 289                                         cerr << "Ignoring comment: " << curr_token().asInput();
 290                         }
 291                         else
 292                                 result += curr_token().asInput();
 293                 }
 294
 295         return make_pair(true, result);
 296 }
 297
 298
 299 string Parser::getArg(char left, char right)
 300 {
 301         return getFullArg(left, right).second;
 302 }
 303
 304
 305 string Parser::getFullOpt()
 306 {
 307         Arg arg = getFullArg('[', ']');
 308         if (arg.first)
 309                 return '[' + arg.second + ']';
 310         return arg.second;
 311 }
 312
 313
 314 string Parser::getOpt()
 315 {
 316         string const res = getArg('[', ']');
 317         return res.empty() ? string() : '[' + res + ']';
 318 }
 319
 320 string Parser::getFullParentheseOpt()
 321 {
 322         Arg arg = getFullArg('(', ')');
 323         if (arg.first)
 324                 return '(' + arg.second + ')';
 325         return arg.second;
 326 }
 327
 328 string const Parser::verbatimEnvironment(string const & name)
 329 {
 330         if (!good())
 331                 return string();
 332
 333         ostringstream os;
 334         for (Token t = get_token(); good(); t = get_token()) {
 335                 if (t.cat() == catBegin) {
 336                         putback();
 337                         os << '{' << verbatim_item() << '}';
 338                 } else if (t.asInput() == "\\begin") {
 339                         string const env = getArg('{', '}');
 340                         os << "\\begin{" << env << '}'
 341                            << verbatimEnvironment(env)
 342                            << "\\end{" << env << '}';
 343                 } else if (t.asInput() == "\\end") {
 344                         string const end = getArg('{', '}');
 345                         if (end != name)
 346                                 cerr << "\\end{" << end
 347                                      << "} does not match \\begin{" << name
 348                                      << "}." << endl;
 349                         return os.str();
 350                 } else
 351                         os << t.asInput();
 352         }
 353         cerr << "unexpected end of input" << endl;
 354         return os.str();
 355 }
 356
 357
 358 void Parser::tokenize(istream & is)
 359 {
 360         static bool init_done = false;
 361
 362         if (!init_done) {
 363                 catInit();
 364                 init_done = true;
 365         }
 366
 367         char c;
 368         while (is.get(c)) {
 369                 //cerr << "reading c: " << c << "\n";
 370
 371                 switch (catcode(c)) {
 372                         case catSpace: {
 373                                 string s(1, c);
 374                                 while (is.get(c) && catcode(c) == catSpace)
 375                                         s += c;
 376                                 if (catcode(c) != catSpace)
 377                                         is.putback(c);
 378                                 push_back(Token(s, catSpace));
 379                                 break;
 380                         }
 381
 382                         case catNewline: {
 383                                 ++lineno_;
 384                                 string s(1, getNewline(is, c));
 385                                 while (is.get(c) && catcode(c) == catNewline) {
 386                                         ++lineno_;
 387                                         s += getNewline(is, c);
 388                                 }
 389                                 if (catcode(c) != catNewline)
 390                                         is.putback(c);
 391                                 push_back(Token(s, catNewline));
 392                                 break;
 393                         }
 394
 395                         case catComment: {
 396                                 // We don't treat "%\n" combinations here specially because
 397                                 // we want to preserve them in the preamble
 398                                 string s;
 399                                 while (is.get(c) && catcode(c) != catNewline)
 400                                         s += c;
 401                                 // handle possible DOS line ending
 402                                 if (catcode(c) == catNewline)
 403                                         c = getNewline(is, c);
 404                                 // Note: The '%' at the beginning and the '\n' at the end
 405                                 // of the comment are not stored.
 406                                 ++lineno_;
 407                                 push_back(Token(s, catComment));
 408                                 break;
 409                         }
 410
 411                         case catEscape: {
 412                                 is.get(c);
 413                                 if (!is) {
 414                                         error("unexpected end of input");
 415                                 } else {
 416                                         string s(1, c);
 417                                         if (catcode(c) == catLetter) {
 418                                                 // collect letters
 419                                                 while (is.get(c) && catcode(c) == catLetter)
 420                                                         s += c;
 421                                                 if (catcode(c) != catLetter)
 422                                                         is.putback(c);
 423                                         }
 424                                         push_back(Token(s, catEscape));
 425                                 }
 426                                 break;
 427                         }
 428
 429                         case catIgnore: {
 430                                 cerr << "ignoring a char: " << int(c) << "\n";
 431                                 break;
 432                         }
 433
 434                         default:
 435                                 push_back(Token(c, catcode(c)));
 436                 }
 437         }
 438 }
 439
 440
 441 void Parser::dump() const
 442 {
 443         cerr << "\nTokens: ";
 444         for (unsigned i = 0; i < tokens_.size(); ++i) {
 445                 if (i == pos_)
 446                         cerr << " <#> ";
 447                 cerr << tokens_[i];
 448         }
 449         cerr << " pos: " << pos_ << "\n";
 450 }
 451
 452
 453 void Parser::error(string const & msg)
 454 {
 455         cerr << "Line ~" << lineno_ << ":  parse error: " << msg << endl;
 456         dump();
 457         //exit(1);
 458 }
 459
 460
 461 string Parser::verbatimOption()
 462 {
 463         string res;
 464         if (next_token().character() == '[') {
 465                 Token t = get_token();
 466                 for (Token t = get_token(); t.character() != ']' && good(); t = get_token()) {
 467                         if (t.cat() == catBegin) {
 468                                 putback();
 469                                 res += '{' + verbatim_item() + '}';
 470                         } else
 471                                 res += t.asString();
 472                 }
 473         }
 474         return res;
 475 }
 476
 477
 478 string Parser::verbatim_item()
 479 {
 480         if (!good())
 481                 error("stream bad");
 482         skip_spaces();
 483         if (next_token().cat() == catBegin) {
 484                 Token t = get_token(); // skip brace
 485                 string res;
 486                 for (Token t = get_token(); t.cat() != catEnd && good(); t = get_token()) {
 487                         if (t.cat() == catBegin) {
 488                                 putback();
 489                                 res += '{' + verbatim_item() + '}';
 490                         }
 491                         else
 492                                 res += t.asInput();
 493                 }
 494                 return res;
 495         }
 496         return get_token().asInput();
 497 }
 498
 499
 500 void Parser::reset()
 501 {
 502         pos_ = 0;
 503 }
 504
 505
 506 void Parser::setCatCode(char c, CatCode cat)
 507 {
 508         theCatcode[(unsigned char)c] = cat;
 509 }
 510
 511
 512 CatCode Parser::getCatCode(char c) const
 513 {
 514         return theCatcode[(unsigned char)c];
 515 }
 516
 517
 518 } // namespace lyx