src/tex2lyx/texparser.C

   1
   2 #include "texparser.h"
   3 #include <iostream>
   4
   5 using std::cerr;
   6 using std::endl;
   7 using std::fill;
   8 using std::ios;
   9 using std::istream;
  10 using std::istringstream;
  11 using std::ostream;
  12 using std::string;
  13 using std::vector;
  14
  15
  16 namespace {
  17
  18 CatCode theCatcode[256];
  19
  20 void skipSpaceTokens(istream & is, char c)
  21 {
  22         // skip trailing spaces
  23         while (catcode(c) == catSpace || catcode(c) == catNewline)
  24                 if (!is.get(c))
  25                         break;
  26         //cerr << "putting back: " << c << "\n";
  27         is.putback(c);
  28 }
  29
  30
  31 void catInit()
  32 {
  33         fill(theCatcode, theCatcode + 256, catOther);
  34         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
  35         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
  36
  37         theCatcode['\\'] = catEscape;
  38         theCatcode['{']  = catBegin;
  39         theCatcode['}']  = catEnd;
  40         theCatcode['$']  = catMath;
  41         theCatcode['&']  = catAlign;
  42         theCatcode[10]   = catNewline;
  43         theCatcode['#']  = catParameter;
  44         theCatcode['^']  = catSuper;
  45         theCatcode['_']  = catSub;
  46         theCatcode['\7f'] = catIgnore;
  47         theCatcode[' ']  = catSpace;
  48         theCatcode['\t'] = catSpace;
  49         theCatcode[13]   = catIgnore;
  50         theCatcode['~']  = catActive;
  51         theCatcode['%']  = catComment;
  52
  53         // This is wrong!
  54         theCatcode['@']  = catLetter;
  55 }
  56
  57 }
  58
  59
  60 //
  61 // catcodes
  62 //
  63
  64 mode_type asMode(mode_type oldmode, string const & str)
  65 {
  66         if (str == "mathmode")
  67                 return MATH_MODE;
  68         if (str == "textmode" || str == "forcetext")
  69                 return TEXT_MODE;
  70         return oldmode;
  71 }
  72
  73
  74 CatCode catcode(unsigned char c)
  75 {
  76         return theCatcode[c];
  77 }
  78
  79
  80
  81 //
  82 // Token
  83 //
  84
  85 ostream & operator<<(ostream & os, Token const & t)
  86 {
  87         if (t.cs().size())
  88                 os << '\\' << t.cs();
  89         else
  90                 os << '[' << t.character() << ',' << t.cat() << ']';
  91         return os;
  92 }
  93
  94
  95 string Token::asString() const
  96 {
  97         return cs_.size() ? cs_ : string(1, char_);
  98 }
  99
 100
 101 string Token::asInput() const
 102 {
 103         return char_ ? string(1, char_) : '\\' + cs_ + ' ';
 104 }
 105
 106
 107 //
 108 // Parser
 109 //
 110
 111
 112 Parser::Parser(istream & is)
 113         : lineno_(0), pos_(0)
 114 {
 115         tokenize(is);
 116 }
 117
 118
 119 void Parser::push_back(Token const & t)
 120 {
 121         tokens_.push_back(t);
 122 }
 123
 124
 125 void Parser::pop_back()
 126 {
 127         tokens_.pop_back();
 128 }
 129
 130
 131 Token const & Parser::prevToken() const
 132 {
 133         static const Token dummy;
 134         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 135 }
 136
 137
 138 Token const & Parser::nextToken() const
 139 {
 140         static const Token dummy;
 141         return good() ? tokens_[pos_] : dummy;
 142 }
 143
 144
 145 Token const & Parser::getToken()
 146 {
 147         static const Token dummy;
 148         //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
 149         return good() ? tokens_[pos_++] : dummy;
 150 }
 151
 152
 153 void Parser::skipSpaces()
 154 {
 155         while (1) {
 156                 if (nextToken().cat() == catSpace || nextToken().cat() == catNewline)
 157                         getToken();
 158                 else if (nextToken().cat() == catComment)
 159                         while (nextToken().cat() != catNewline)
 160                                 getToken();
 161                 else
 162                         break;
 163         }
 164 }
 165
 166
 167 void Parser::putback()
 168 {
 169         --pos_;
 170 }
 171
 172
 173 bool Parser::good() const
 174 {
 175         return pos_ < tokens_.size();
 176 }
 177
 178
 179 char Parser::getChar()
 180 {
 181         if (!good())
 182                 error("The input stream is not well...");
 183         return tokens_[pos_++].character();
 184 }
 185
 186
 187 string Parser::getArg(char left, char right)
 188 {
 189         skipSpaces();
 190
 191         string result;
 192         char c = getChar();
 193
 194         if (c != left)
 195                 putback();
 196         else
 197                 while ((c = getChar()) != right && good())
 198                         result += c;
 199
 200         return result;
 201 }
 202
 203
 204 string Parser::getOpt()
 205 {
 206         string res = getArg('[', ']');
 207         return res.size() ? '[' + res + ']' : string();
 208 }
 209
 210
 211 void Parser::tokenize(istream & is)
 212 {
 213         static bool init_done = false;
 214
 215         if (!init_done) {
 216                 catInit();
 217                 init_done = true;
 218         }
 219
 220         char c;
 221         while (is.get(c)) {
 222                 //cerr << "reading c: " << c << "\n";
 223
 224                 switch (catcode(c)) {
 225                         case catNewline: {
 226                                 ++lineno_;
 227                                 is.get(c);
 228                                 if (catcode(c) == catNewline)
 229                                         push_back(Token("par"));
 230                                 else {
 231                                         push_back(Token('\n', catNewline));
 232                                         is.putback(c);
 233                                 }
 234                                 break;
 235                         }
 236
 237 /*
 238                         case catComment: {
 239                                 while (is.get(c) && catcode(c) != catNewline)
 240                                         ;
 241                                 ++lineno_;
 242                                 break;
 243                         }
 244 */
 245
 246                         case catEscape: {
 247                                 is.get(c);
 248                                 if (!is) {
 249                                         error("unexpected end of input");
 250                                 } else {
 251                                         string s(1, c);
 252                                         if (catcode(c) == catLetter) {
 253                                                 // collect letters
 254                                                 while (is.get(c) && catcode(c) == catLetter)
 255                                                         s += c;
 256                                                 skipSpaceTokens(is, c);
 257                                         }
 258                                         push_back(Token(s));
 259                                 }
 260                                 break;
 261                         }
 262
 263                         case catSuper:
 264                         case catSub: {
 265                                 push_back(Token(c, catcode(c)));
 266                                 is.get(c);
 267                                 skipSpaceTokens(is, c);
 268                                 break;
 269                         }
 270
 271                         case catIgnore: {
 272                                 if (c != 13)
 273                                         cerr << "ignoring a char: " << int(c) << "\n";
 274                                 break;
 275                         }
 276
 277                         default:
 278                                 push_back(Token(c, catcode(c)));
 279                 }
 280         }
 281 }
 282
 283
 284 void Parser::dump() const
 285 {
 286         cerr << "\nTokens: ";
 287         for (unsigned i = 0; i < tokens_.size(); ++i) {
 288                 if (i == pos_)
 289                         cerr << " <#> ";
 290                 cerr << tokens_[i];
 291         }
 292         cerr << " pos: " << pos_ << "\n";
 293 }
 294
 295
 296 void Parser::error(string const & msg)
 297 {
 298         cerr << "Line ~" << lineno_ << ":  parse error: " << msg << endl;
 299         dump();
 300         //exit(1);
 301 }
 302
 303
 304 string Parser::verbatimOption()
 305 {
 306         string res;
 307         if (nextToken().character() == '[') {
 308                 Token t = getToken();
 309                 for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
 310                         if (t.cat() == catBegin) {
 311                                 putback();
 312                                 res += '{' + verbatimItem() + '}';
 313                         } else
 314                                 res += t.asString();
 315                 }
 316         }
 317         return res;
 318 }
 319
 320
 321 string Parser::verbatimItem()
 322 {
 323         if (!good())
 324                 error("stream bad");
 325         skipSpaces();
 326         if (nextToken().cat() == catBegin) {
 327                 Token t = getToken(); // skip brace
 328                 string res;
 329                 for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
 330                         if (t.cat() == catBegin) {
 331                                 putback();
 332                                 res += '{' + verbatimItem() + '}';
 333                         }
 334                         else
 335                                 res += t.asInput();
 336                 }
 337                 return res;
 338         }
 339         return getToken().asInput();
 340 }
 341
 342
 343 void Parser::setCatCode(char c, CatCode cat)
 344 {
 345         theCatcode[c] = cat;
 346 }
 347
 348
 349 CatCode Parser::getCatCode(char c) const
 350 {
 351         return theCatcode[c];
 352 }