src/tex2lyx/texparser.C

   1 /**
   2  * \file texparser.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  */
  10
  11 #include <config.h>
  12
  13 #include "texparser.h"
  14
  15 #include <iostream>
  16 #include <sstream>
  17
  18 using std::cerr;
  19 using std::endl;
  20 using std::fill;
  21 using std::istream;
  22 using std::istringstream;
  23 using std::ostream;
  24 using std::string;
  25
  26
  27 namespace {
  28
  29 CatCode theCatcode[256];
  30
  31 void catInit()
  32 {
  33         fill(theCatcode, theCatcode + 256, catOther);
  34         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
  35         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
  36
  37         theCatcode[int('\\')] = catEscape;
  38         theCatcode[int('{')]  = catBegin;
  39         theCatcode[int('}')]  = catEnd;
  40         theCatcode[int('$')]  = catMath;
  41         theCatcode[int('&')]  = catAlign;
  42         theCatcode[10]   = catNewline;
  43         theCatcode[int('#')]  = catParameter;
  44         theCatcode[int('^')]  = catSuper;
  45         theCatcode[int('_')]  = catSub;
  46         theCatcode[0x7f] = catIgnore;
  47         theCatcode[int(' ')]  = catSpace;
  48         theCatcode[int('\t')] = catSpace;
  49         theCatcode[13]   = catIgnore;
  50         theCatcode[int('~')]  = catActive;
  51         theCatcode[int('%')]  = catComment;
  52
  53         // This is wrong!
  54         theCatcode[int('@')]  = catLetter;
  55 }
  56
  57 }
  58
  59
  60 //
  61 // catcodes
  62 //
  63
  64 mode_type asMode(mode_type oldmode, string const & str)
  65 {
  66         if (str == "mathmode")
  67                 return MATH_MODE;
  68         if (str == "textmode" || str == "forcetext")
  69                 return TEXT_MODE;
  70         return oldmode;
  71 }
  72
  73
  74 CatCode catcode(unsigned char c)
  75 {
  76         return theCatcode[c];
  77 }
  78
  79
  80
  81 //
  82 // Token
  83 //
  84
  85 ostream & operator<<(ostream & os, Token const & t)
  86 {
  87         if (t.cat() == catComment)
  88                 os << '%' << t.cs() << '\n';
  89         else if (t.cat() == catSpace)
  90                 os << t.cs();
  91         else if (t.cat() == catEscape)
  92                 os << '\\' << t.cs() << ' ';
  93         else if (t.cat() == catLetter)
  94                 os << t.character();
  95         else if (t.cat() == catNewline)
  96                 os << "[" << t.cs().size() << "\\n," << t.cat() << "]\n";
  97         else
  98                 os << '[' << t.character() << ',' << t.cat() << ']';
  99         return os;
 100 }
 101
 102
 103 string Token::asString() const
 104 {
 105         return cs_.size() ? cs_ : string(1, char_);
 106 }
 107
 108
 109 string Token::asInput() const
 110 {
 111         if (cat_ == catComment)
 112                 return '%' + cs_ + '\n';
 113         if (cat_ == catSpace || cat_ == catNewline)
 114                 return cs_;
 115         return char_ ? string(1, char_) : '\\' + cs_;
 116 }
 117
 118
 119 //
 120 // Parser
 121 //
 122
 123
 124 Parser::Parser(istream & is)
 125         : lineno_(0), pos_(0)
 126 {
 127         tokenize(is);
 128 }
 129
 130
 131 Parser::Parser(string const & s)
 132         : lineno_(0), pos_(0)
 133 {
 134         istringstream is(s);
 135         tokenize(is);
 136 }
 137
 138
 139 void Parser::push_back(Token const & t)
 140 {
 141         tokens_.push_back(t);
 142 }
 143
 144
 145 void Parser::pop_back()
 146 {
 147         tokens_.pop_back();
 148 }
 149
 150
 151 Token const & Parser::prev_token() const
 152 {
 153         static const Token dummy;
 154         return pos_ > 1 ? tokens_[pos_ - 2] : dummy;
 155 }
 156
 157
 158 Token const & Parser::curr_token() const
 159 {
 160         static const Token dummy;
 161         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 162 }
 163
 164
 165 Token const & Parser::next_token() const
 166 {
 167         static const Token dummy;
 168         return good() ? tokens_[pos_] : dummy;
 169 }
 170
 171
 172 Token const & Parser::get_token()
 173 {
 174         static const Token dummy;
 175         //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
 176         return good() ? tokens_[pos_++] : dummy;
 177 }
 178
 179
 180 void Parser::skip_spaces(bool skip_comments)
 181 {
 182         // We just silently return if we have no more tokens.
 183         // skip_spaces() should be callable at any time,
 184         // the caller must check p::good() anyway.
 185         while (good()) {
 186                 if ( next_token().cat() == catSpace ||
 187                     (next_token().cat() == catNewline && next_token().cs().size() == 1) ||
 188                      next_token().cat() == catComment && next_token().cs().empty())
 189                         get_token();
 190                 else if (skip_comments && next_token().cat() == catComment)
 191                         cerr << "  Ignoring comment: " << get_token().asInput();
 192                 else
 193                         break;
 194         }
 195 }
 196
 197
 198 void Parser::unskip_spaces(bool skip_comments)
 199 {
 200         while (pos_ > 0) {
 201                 if ( curr_token().cat() == catSpace ||
 202                     (curr_token().cat() == catNewline && curr_token().cs().size() == 1))
 203                         putback();
 204                 else if (skip_comments && curr_token().cat() == catComment) {
 205                         // TODO: Get rid of this
 206                         cerr << "Unignoring comment: " << curr_token().asInput();
 207                         putback();
 208                 }
 209                 else
 210                         break;
 211         }
 212 }
 213
 214
 215 void Parser::putback()
 216 {
 217         --pos_;
 218 }
 219
 220
 221 bool Parser::good() const
 222 {
 223         return pos_ < tokens_.size();
 224 }
 225
 226
 227 char Parser::getChar()
 228 {
 229         if (!good())
 230                 error("The input stream is not well...");
 231         return tokens_[pos_++].character();
 232 }
 233
 234
 235 string Parser::getArg(char left, char right)
 236 {
 237         skip_spaces(true);
 238
 239         // This is needed if a partial file ends with a command without arguments,
 240         // e. g. \medskip
 241         if (! good())
 242                 return string();
 243
 244         string result;
 245         char c = getChar();
 246
 247         if (c != left)
 248                 putback();
 249         else
 250                 while ((c = getChar()) != right && good()) {
 251                         // Ignore comments
 252                         if (curr_token().cat() == catComment) {
 253                                 if (curr_token().cs().size())
 254                                         cerr << "Ignoring comment: " << curr_token().asInput();
 255                         }
 256                         else if (curr_token().cat() == catSpace || curr_token().cat() == catNewline)
 257                                 result += curr_token().cs();
 258                         else
 259                                 result += c;
 260                 }
 261
 262         return result;
 263 }
 264
 265
 266 string Parser::getOpt()
 267 {
 268         string const res = getArg('[', ']');
 269         return res.size() ? '[' + res + ']' : string();
 270 }
 271
 272
 273 void Parser::tokenize(istream & is)
 274 {
 275         static bool init_done = false;
 276
 277         if (!init_done) {
 278                 catInit();
 279                 init_done = true;
 280         }
 281
 282         char c;
 283         while (is.get(c)) {
 284                 //cerr << "reading c: " << c << "\n";
 285
 286                 switch (catcode(c)) {
 287                         case catSpace: {
 288                                 string s(1, c);
 289                                 while (is.get(c) && catcode(c) == catSpace)
 290                                         s += c;
 291                                 if (catcode(c) != catSpace)
 292                                         is.putback(c);
 293                                 push_back(Token(s, catSpace));
 294                                 break;
 295                         }
 296
 297                         case catNewline: {
 298                                 ++lineno_;
 299                                 string s(1, c);
 300                                 while (is.get(c) && catcode(c) == catNewline) {
 301                                         ++lineno_;
 302                                         s += c;
 303                                 }
 304                                 if (catcode(c) != catNewline)
 305                                         is.putback(c);
 306                                 push_back(Token(s, catNewline));
 307                                 break;
 308                         }
 309
 310                         case catComment: {
 311                                 // We don't treat "%\n" combinations here specially because
 312                                 // we want to preserve them in the preamble
 313                                 string s;
 314                                 while (is.get(c) && catcode(c) != catNewline)
 315                                         s += c;
 316                                 // Note: The '%' at the beginning and the '\n' at the end
 317                                 // of the comment are not stored.
 318                                 ++lineno_;
 319                                 push_back(Token(s, catComment));
 320                                 break;
 321                         }
 322
 323                         case catEscape: {
 324                                 is.get(c);
 325                                 if (!is) {
 326                                         error("unexpected end of input");
 327                                 } else {
 328                                         string s(1, c);
 329                                         if (catcode(c) == catLetter) {
 330                                                 // collect letters
 331                                                 while (is.get(c) && catcode(c) == catLetter)
 332                                                         s += c;
 333                                                 if (catcode(c) != catLetter)
 334                                                         is.putback(c);
 335                                         }
 336                                         push_back(Token(s, catEscape));
 337                                 }
 338                                 break;
 339                         }
 340
 341                         case catIgnore: {
 342                                 if (c != 13)
 343                                         cerr << "ignoring a char: " << int(c) << "\n";
 344                                 break;
 345                         }
 346
 347                         default:
 348                                 push_back(Token(c, catcode(c)));
 349                 }
 350         }
 351 }
 352
 353
 354 void Parser::dump() const
 355 {
 356         cerr << "\nTokens: ";
 357         for (unsigned i = 0; i < tokens_.size(); ++i) {
 358                 if (i == pos_)
 359                         cerr << " <#> ";
 360                 cerr << tokens_[i];
 361         }
 362         cerr << " pos: " << pos_ << "\n";
 363 }
 364
 365
 366 void Parser::error(string const & msg)
 367 {
 368         cerr << "Line ~" << lineno_ << ":  parse error: " << msg << endl;
 369         dump();
 370         //exit(1);
 371 }
 372
 373
 374 string Parser::verbatimOption()
 375 {
 376         string res;
 377         if (next_token().character() == '[') {
 378                 Token t = get_token();
 379                 for (Token t = get_token(); t.character() != ']' && good(); t = get_token()) {
 380                         if (t.cat() == catBegin) {
 381                                 putback();
 382                                 res += '{' + verbatim_item() + '}';
 383                         } else
 384                                 res += t.asString();
 385                 }
 386         }
 387         return res;
 388 }
 389
 390
 391 string Parser::verbatim_item()
 392 {
 393         if (!good())
 394                 error("stream bad");
 395         skip_spaces();
 396         if (next_token().cat() == catBegin) {
 397                 Token t = get_token(); // skip brace
 398                 string res;
 399                 for (Token t = get_token(); t.cat() != catEnd && good(); t = get_token()) {
 400                         if (t.cat() == catBegin) {
 401                                 putback();
 402                                 res += '{' + verbatim_item() + '}';
 403                         }
 404                         else
 405                                 res += t.asInput();
 406                 }
 407                 return res;
 408         }
 409         return get_token().asInput();
 410 }
 411
 412
 413 void Parser::reset()
 414 {
 415         pos_ = 0;
 416 }
 417
 418
 419 void Parser::setCatCode(char c, CatCode cat)
 420 {
 421         theCatcode[(unsigned char)c] = cat;
 422 }
 423
 424
 425 CatCode Parser::getCatCode(char c) const
 426 {
 427         return theCatcode[(unsigned char)c];
 428 }