src/tex2lyx/texparser.C

   1
   2 #include "texparser.h"
   3 #include <iostream>
   4
   5 using std::cerr;
   6 using std::endl;
   7 using std::fill;
   8 using std::ios;
   9 using std::istream;
  10 using std::istringstream;
  11 using std::ostream;
  12 using std::string;
  13 using std::vector;
  14
  15
  16 namespace {
  17
  18 CatCode theCatcode[256];
  19
  20 void skipSpaceTokens(istream & is, char c)
  21 {
  22         // skip trailing spaces
  23         while (catcode(c) == catSpace || catcode(c) == catNewline)
  24                 if (!is.get(c))
  25                         break;
  26         //cerr << "putting back: " << c << "\n";
  27         is.putback(c);
  28 }
  29
  30
  31 void catInit()
  32 {
  33         fill(theCatcode, theCatcode + 256, catOther);
  34         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
  35         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
  36
  37         theCatcode['\\'] = catEscape;
  38         theCatcode['{']  = catBegin;
  39         theCatcode['}']  = catEnd;
  40         theCatcode['$']  = catMath;
  41         theCatcode['&']  = catAlign;
  42         theCatcode[10]   = catNewline;
  43         theCatcode['#']  = catParameter;
  44         theCatcode['^']  = catSuper;
  45         theCatcode['_']  = catSub;
  46         theCatcode['\7f'] = catIgnore;
  47         theCatcode[' ']  = catSpace;
  48         theCatcode['\t'] = catSpace;
  49         theCatcode[13]   = catIgnore;
  50         theCatcode['~']  = catActive;
  51         theCatcode['%']  = catComment;
  52 }
  53
  54 }
  55
  56
  57 //
  58 // catcodes
  59 //
  60
  61 mode_type asMode(mode_type oldmode, string const & str)
  62 {
  63         if (str == "mathmode")
  64                 return MATH_MODE;
  65         if (str == "textmode" || str == "forcetext")
  66                 return TEXT_MODE;
  67         return oldmode;
  68 }
  69
  70
  71 CatCode catcode(unsigned char c)
  72 {
  73         return theCatcode[c];
  74 }
  75
  76
  77
  78 //
  79 // Token
  80 //
  81
  82 ostream & operator<<(ostream & os, Token const & t)
  83 {
  84         if (t.cs().size())
  85                 os << '\\' << t.cs();
  86         else
  87                 os << '[' << t.character() << ',' << t.cat() << ']';
  88         return os;
  89 }
  90
  91
  92 string Token::asString() const
  93 {
  94         return cs_.size() ? cs_ : string(1, char_);
  95 }
  96
  97
  98 string Token::asInput() const
  99 {
 100         return char_ ? string(1, char_) : '\\' + cs_ + ' ';
 101 }
 102
 103
 104 //
 105 // Parser
 106 //
 107
 108
 109 Parser::Parser(istream & is)
 110         : lineno_(0), pos_(0)
 111 {
 112         tokenize(is);
 113 }
 114
 115
 116 void Parser::push_back(Token const & t)
 117 {
 118         tokens_.push_back(t);
 119 }
 120
 121
 122 void Parser::pop_back()
 123 {
 124         tokens_.pop_back();
 125 }
 126
 127
 128 Token const & Parser::prevToken() const
 129 {
 130         static const Token dummy;
 131         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 132 }
 133
 134
 135 Token const & Parser::nextToken() const
 136 {
 137         static const Token dummy;
 138         return good() ? tokens_[pos_] : dummy;
 139 }
 140
 141
 142 Token const & Parser::getToken()
 143 {
 144         static const Token dummy;
 145         //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
 146         return good() ? tokens_[pos_++] : dummy;
 147 }
 148
 149
 150 void Parser::skipSpaces()
 151 {
 152         while (nextToken().cat() == catSpace || nextToken().cat() == catNewline)
 153                 getToken();
 154 }
 155
 156
 157 void Parser::putback()
 158 {
 159         --pos_;
 160 }
 161
 162
 163 bool Parser::good() const
 164 {
 165         return pos_ < tokens_.size();
 166 }
 167
 168
 169 char Parser::getChar()
 170 {
 171         if (!good())
 172                 error("The input stream is not well...");
 173         return tokens_[pos_++].character();
 174 }
 175
 176
 177 string Parser::getArg(char left, char right)
 178 {
 179         skipSpaces();
 180
 181         string result;
 182         char c = getChar();
 183
 184         if (c != left)
 185                 putback();
 186         else
 187                 while ((c = getChar()) != right && good())
 188                         result += c;
 189
 190         return result;
 191 }
 192
 193
 194 string Parser::getOpt()
 195 {
 196         string res = getArg('[', ']');
 197         return res.size() ? '[' + res + ']' : string();
 198 }
 199
 200
 201 void Parser::tokenize(istream & is)
 202 {
 203         static bool init_done = false;
 204
 205         if (!init_done) {
 206                 catInit();
 207                 init_done = true;
 208         }
 209
 210         char c;
 211         while (is.get(c)) {
 212                 //cerr << "reading c: " << c << "\n";
 213
 214                 switch (catcode(c)) {
 215                         case catNewline: {
 216                                 ++lineno_;
 217                                 is.get(c);
 218                                 if (catcode(c) == catNewline)
 219                                         push_back(Token("par"));
 220                                 else {
 221                                         push_back(Token('\n', catNewline));
 222                                         is.putback(c);
 223                                 }
 224                                 break;
 225                         }
 226
 227 /*
 228                         case catComment: {
 229                                 while (is.get(c) && catcode(c) != catNewline)
 230                                         ;
 231                                 ++lineno_;
 232                                 break;
 233                         }
 234 */
 235
 236                         case catEscape: {
 237                                 is.get(c);
 238                                 if (!is) {
 239                                         error("unexpected end of input");
 240                                 } else {
 241                                         string s(1, c);
 242                                         if (catcode(c) == catLetter) {
 243                                                 // collect letters
 244                                                 while (is.get(c) && catcode(c) == catLetter)
 245                                                         s += c;
 246                                                 skipSpaceTokens(is, c);
 247                                         }
 248                                         push_back(Token(s));
 249                                 }
 250                                 break;
 251                         }
 252
 253                         case catSuper:
 254                         case catSub: {
 255                                 push_back(Token(c, catcode(c)));
 256                                 is.get(c);
 257                                 skipSpaceTokens(is, c);
 258                                 break;
 259                         }
 260
 261                         case catIgnore: {
 262                                 if (c != 13)
 263                                         cerr << "ignoring a char: " << int(c) << "\n";
 264                                 break;
 265                         }
 266
 267                         default:
 268                                 push_back(Token(c, catcode(c)));
 269                 }
 270         }
 271 }
 272
 273
 274 void Parser::dump() const
 275 {
 276         cerr << "\nTokens: ";
 277         for (unsigned i = 0; i < tokens_.size(); ++i) {
 278                 if (i == pos_)
 279                         cerr << " <#> ";
 280                 cerr << tokens_[i];
 281         }
 282         cerr << " pos: " << pos_ << "\n";
 283 }
 284
 285
 286 void Parser::error(string const & msg)
 287 {
 288         cerr << "Line ~" << lineno_ << ":  parse error: " << msg << endl;
 289         dump();
 290         //exit(1);
 291 }
 292
 293
 294 string Parser::verbatimOption()
 295 {
 296         string res;
 297         if (nextToken().character() == '[') {
 298                 Token t = getToken();
 299                 for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
 300                         if (t.cat() == catBegin) {
 301                                 putback();
 302                                 res += '{' + verbatimItem() + '}';
 303                         } else
 304                                 res += t.asString();
 305                 }
 306         }
 307         return res;
 308 }
 309
 310
 311 string Parser::verbatimItem()
 312 {
 313         if (!good())
 314                 error("stream bad");
 315         if (nextToken().cat() == catBegin) {
 316                 Token t = getToken(); // skip brace
 317                 string res;
 318                 for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
 319                         if (t.cat() == catBegin) {
 320                                 putback();
 321                                 res += '{' + verbatimItem() + '}';
 322                         }
 323                         else
 324                                 res += t.asInput();
 325                 }
 326                 return res;
 327         }
 328         return getToken().asInput();
 329 }
 330
 331
 332 void Parser::setCatCode(char c, CatCode cat)
 333 {
 334         theCatcode[c] = cat;
 335 }
 336
 337
 338 CatCode Parser::getCatCode(char c) const
 339 {
 340         return theCatcode[c];
 341 }