src/tex2lyx/texparser.C

   1
   2 #include <config.h>
   3
   4 #include "texparser.h"
   5 #include "Lsstream.h"
   6
   7 #include <iostream>
   8
   9 using std::cerr;
  10 using std::endl;
  11 using std::fill;
  12 using std::ios;
  13 using std::istream;
  14 using std::istringstream;
  15 using std::ostream;
  16 using std::vector;
  17
  18
  19 namespace {
  20
  21 CatCode theCatcode[256];
  22
  23 void skipSpaceTokens(istream & is, char c)
  24 {
  25         // skip trailing spaces
  26         while (catcode(c) == catSpace || catcode(c) == catNewline)
  27                 if (!is.get(c))
  28                         break;
  29         //cerr << "putting back: " << c << "\n";
  30         is.putback(c);
  31 }
  32
  33
  34 void catInit()
  35 {
  36         fill(theCatcode, theCatcode + 256, catOther);
  37         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
  38         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
  39
  40         theCatcode['\\'] = catEscape;
  41         theCatcode['{']  = catBegin;
  42         theCatcode['}']  = catEnd;
  43         theCatcode['$']  = catMath;
  44         theCatcode['&']  = catAlign;
  45         theCatcode[10]   = catNewline;
  46         theCatcode['#']  = catParameter;
  47         theCatcode['^']  = catSuper;
  48         theCatcode['_']  = catSub;
  49         theCatcode['\7f'] = catIgnore;
  50         theCatcode[' ']  = catSpace;
  51         theCatcode['\t'] = catSpace;
  52         theCatcode[13]   = catIgnore;
  53         theCatcode['~']  = catActive;
  54         theCatcode['%']  = catComment;
  55
  56         // This is wrong!
  57         theCatcode['@']  = catLetter;
  58 }
  59
  60 }
  61
  62
  63 //
  64 // catcodes
  65 //
  66
  67 mode_type asMode(mode_type oldmode, string const & str)
  68 {
  69         if (str == "mathmode")
  70                 return MATH_MODE;
  71         if (str == "textmode" || str == "forcetext")
  72                 return TEXT_MODE;
  73         return oldmode;
  74 }
  75
  76
  77 CatCode catcode(unsigned char c)
  78 {
  79         return theCatcode[c];
  80 }
  81
  82
  83
  84 //
  85 // Token
  86 //
  87
  88 ostream & operator<<(ostream & os, Token const & t)
  89 {
  90         if (t.cs().size())
  91                 os << '\\' << t.cs() << ' ';
  92         else if (t.cat() == catLetter)
  93                 os << t.character();
  94         else if (t.cat() == catNewline)
  95                 os << "[\\n," << t.cat() << "]\n";
  96         else
  97                 os << '[' << t.character() << ',' << t.cat() << ']';
  98         return os;
  99 }
 100
 101
 102 string Token::asString() const
 103 {
 104         return cs_.size() ? cs_ : string(1, char_);
 105 }
 106
 107
 108 string Token::asInput() const
 109 {
 110         return char_ ? string(1, char_) : '\\' + cs_ + ' ';
 111 }
 112
 113
 114 //
 115 // Parser
 116 //
 117
 118
 119 Parser::Parser(istream & is)
 120         : lineno_(0), pos_(0)
 121 {
 122         tokenize(is);
 123 }
 124
 125
 126 Parser::Parser(string const & s)
 127         : lineno_(0), pos_(0)
 128 {
 129         istringstream is(STRCONV(s));
 130         tokenize(is);
 131 }
 132
 133
 134 void Parser::push_back(Token const & t)
 135 {
 136         tokens_.push_back(t);
 137 }
 138
 139
 140 void Parser::pop_back()
 141 {
 142         tokens_.pop_back();
 143 }
 144
 145
 146 Token const & Parser::prev_token() const
 147 {
 148         static const Token dummy;
 149         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 150 }
 151
 152
 153 Token const & Parser::next_token() const
 154 {
 155         static const Token dummy;
 156         return good() ? tokens_[pos_] : dummy;
 157 }
 158
 159
 160 Token const & Parser::get_token()
 161 {
 162         static const Token dummy;
 163         //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
 164         return good() ? tokens_[pos_++] : dummy;
 165 }
 166
 167
 168 void Parser::skip_spaces()
 169 {
 170         while (1) {
 171                 if (next_token().cat() == catSpace || next_token().cat() == catNewline)
 172                         get_token();
 173                 else if (next_token().cat() == catComment)
 174                         while (next_token().cat() != catNewline)
 175                                 get_token();
 176                 else
 177                         break;
 178         }
 179 }
 180
 181
 182 void Parser::putback()
 183 {
 184         --pos_;
 185 }
 186
 187
 188 bool Parser::good() const
 189 {
 190         return pos_ < tokens_.size();
 191 }
 192
 193
 194 char Parser::getChar()
 195 {
 196         if (!good())
 197                 error("The input stream is not well...");
 198         return tokens_[pos_++].character();
 199 }
 200
 201
 202 string Parser::getArg(char left, char right)
 203 {
 204         skip_spaces();
 205
 206         string result;
 207         char c = getChar();
 208
 209         if (c != left)
 210                 putback();
 211         else
 212                 while ((c = getChar()) != right && good())
 213                         result += c;
 214
 215         return result;
 216 }
 217
 218
 219 string Parser::getOpt()
 220 {
 221         string res = getArg('[', ']');
 222         return res.size() ? '[' + res + ']' : string();
 223 }
 224
 225
 226 void Parser::tokenize(istream & is)
 227 {
 228         static bool init_done = false;
 229
 230         if (!init_done) {
 231                 catInit();
 232                 init_done = true;
 233         }
 234
 235         char c;
 236         while (is.get(c)) {
 237                 //cerr << "reading c: " << c << "\n";
 238
 239                 switch (catcode(c)) {
 240                         case catNewline: {
 241                                 ++lineno_;
 242                                 is.get(c);
 243                                 if (catcode(c) == catNewline) {
 244                                         //do {
 245                                                 is.get(c);
 246                                         //} while (catcode(c) == catNewline);
 247                                         push_back(Token("par"));
 248                                 } else {
 249                                         push_back(Token('\n', catNewline));
 250                                 }
 251                                 is.putback(c);
 252                                 break;
 253                         }
 254
 255 /*
 256                         case catComment: {
 257                                 while (is.get(c) && catcode(c) != catNewline)
 258                                         ;
 259                                 ++lineno_;
 260                                 break;
 261                         }
 262 */
 263
 264                         case catEscape: {
 265                                 is.get(c);
 266                                 if (!is) {
 267                                         error("unexpected end of input");
 268                                 } else {
 269                                         string s(1, c);
 270                                         if (catcode(c) == catLetter) {
 271                                                 // collect letters
 272                                                 while (is.get(c) && catcode(c) == catLetter)
 273                                                         s += c;
 274                                                 skipSpaceTokens(is, c);
 275                                         }
 276                                         push_back(Token(s));
 277                                 }
 278                                 break;
 279                         }
 280
 281                         case catSuper:
 282                         case catSub: {
 283                                 push_back(Token(c, catcode(c)));
 284                                 is.get(c);
 285                                 skipSpaceTokens(is, c);
 286                                 break;
 287                         }
 288
 289                         case catIgnore: {
 290                                 if (c != 13)
 291                                         cerr << "ignoring a char: " << int(c) << "\n";
 292                                 break;
 293                         }
 294
 295                         default:
 296                                 push_back(Token(c, catcode(c)));
 297                 }
 298         }
 299 }
 300
 301
 302 void Parser::dump() const
 303 {
 304         cerr << "\nTokens: ";
 305         for (unsigned i = 0; i < tokens_.size(); ++i) {
 306                 if (i == pos_)
 307                         cerr << " <#> ";
 308                 cerr << tokens_[i];
 309         }
 310         cerr << " pos: " << pos_ << "\n";
 311 }
 312
 313
 314 void Parser::error(string const & msg)
 315 {
 316         cerr << "Line ~" << lineno_ << ":  parse error: " << msg << endl;
 317         dump();
 318         //exit(1);
 319 }
 320
 321
 322 string Parser::verbatimOption()
 323 {
 324         string res;
 325         if (next_token().character() == '[') {
 326                 Token t = get_token();
 327                 for (Token t = get_token(); t.character() != ']' && good(); t = get_token()) {
 328                         if (t.cat() == catBegin) {
 329                                 putback();
 330                                 res += '{' + verbatim_item() + '}';
 331                         } else
 332                                 res += t.asString();
 333                 }
 334         }
 335         return res;
 336 }
 337
 338
 339 string Parser::verbatim_item()
 340 {
 341         if (!good())
 342                 error("stream bad");
 343         skip_spaces();
 344         if (next_token().cat() == catBegin) {
 345                 Token t = get_token(); // skip brace
 346                 string res;
 347                 for (Token t = get_token(); t.cat() != catEnd && good(); t = get_token()) {
 348                         if (t.cat() == catBegin) {
 349                                 putback();
 350                                 res += '{' + verbatim_item() + '}';
 351                         }
 352                         else
 353                                 res += t.asInput();
 354                 }
 355                 return res;
 356         }
 357         return get_token().asInput();
 358 }
 359
 360
 361 void Parser::setCatCode(char c, CatCode cat)
 362 {
 363         theCatcode[c] = cat;
 364 }
 365
 366
 367 CatCode Parser::getCatCode(char c) const
 368 {
 369         return theCatcode[c];
 370 }