src/tex2lyx/texparser.C

   1
   2 #include "texparser.h"
   3 #include <iostream>
   4
   5 using std::cerr;
   6 using std::endl;
   7 using std::fill;
   8 using std::ios;
   9 using std::istream;
  10 using std::istringstream;
  11 using std::ostream;
  12 using std::string;
  13 using std::vector;
  14
  15
  16 namespace {
  17
  18 CatCode theCatcode[256];
  19
  20 void skipSpaceTokens(istream & is, char c)
  21 {
  22         // skip trailing spaces
  23         while (catcode(c) == catSpace || catcode(c) == catNewline)
  24                 if (!is.get(c))
  25                         break;
  26         //cerr << "putting back: " << c << "\n";
  27         is.putback(c);
  28 }
  29
  30
  31 void catInit()
  32 {
  33         fill(theCatcode, theCatcode + 256, catOther);
  34         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
  35         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
  36
  37         theCatcode['\\'] = catEscape;
  38         theCatcode['{']  = catBegin;
  39         theCatcode['}']  = catEnd;
  40         theCatcode['$']  = catMath;
  41         theCatcode['&']  = catAlign;
  42         theCatcode[10]   = catNewline;
  43         theCatcode['#']  = catParameter;
  44         theCatcode['^']  = catSuper;
  45         theCatcode['_']  = catSub;
  46         theCatcode['\7f'] = catIgnore;
  47         theCatcode[' ']  = catSpace;
  48         theCatcode['\t'] = catSpace;
  49         theCatcode[13]   = catIgnore;
  50         theCatcode['~']  = catActive;
  51         theCatcode['%']  = catComment;
  52
  53         // This is wrong!
  54         theCatcode['@']  = catLetter;
  55 }
  56
  57 }
  58
  59
  60 //
  61 // catcodes
  62 //
  63
  64 mode_type asMode(mode_type oldmode, string const & str)
  65 {
  66         if (str == "mathmode")
  67                 return MATH_MODE;
  68         if (str == "textmode" || str == "forcetext")
  69                 return TEXT_MODE;
  70         return oldmode;
  71 }
  72
  73
  74 CatCode catcode(unsigned char c)
  75 {
  76         return theCatcode[c];
  77 }
  78
  79
  80
  81 //
  82 // Token
  83 //
  84
  85 ostream & operator<<(ostream & os, Token const & t)
  86 {
  87         if (t.cs().size())
  88                 os << '\\' << t.cs() << ' ';
  89         else if (t.cat() == catLetter)
  90                 os << t.character();
  91         else
  92                 os << '[' << t.character() << ',' << t.cat() << ']';
  93         return os;
  94 }
  95
  96
  97 string Token::asString() const
  98 {
  99         return cs_.size() ? cs_ : string(1, char_);
 100 }
 101
 102
 103 string Token::asInput() const
 104 {
 105         return char_ ? string(1, char_) : '\\' + cs_ + ' ';
 106 }
 107
 108
 109 //
 110 // Parser
 111 //
 112
 113
 114 Parser::Parser(istream & is)
 115         : lineno_(0), pos_(0)
 116 {
 117         tokenize(is);
 118 }
 119
 120
 121 void Parser::push_back(Token const & t)
 122 {
 123         tokens_.push_back(t);
 124 }
 125
 126
 127 void Parser::pop_back()
 128 {
 129         tokens_.pop_back();
 130 }
 131
 132
 133 Token const & Parser::prevToken() const
 134 {
 135         static const Token dummy;
 136         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 137 }
 138
 139
 140 Token const & Parser::nextToken() const
 141 {
 142         static const Token dummy;
 143         return good() ? tokens_[pos_] : dummy;
 144 }
 145
 146
 147 Token const & Parser::getToken()
 148 {
 149         static const Token dummy;
 150         //cerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
 151         return good() ? tokens_[pos_++] : dummy;
 152 }
 153
 154
 155 void Parser::skipSpaces()
 156 {
 157         while (1) {
 158                 if (nextToken().cat() == catSpace || nextToken().cat() == catNewline)
 159                         getToken();
 160                 else if (nextToken().cat() == catComment)
 161                         while (nextToken().cat() != catNewline)
 162                                 getToken();
 163                 else
 164                         break;
 165         }
 166 }
 167
 168
 169 void Parser::putback()
 170 {
 171         --pos_;
 172 }
 173
 174
 175 bool Parser::good() const
 176 {
 177         return pos_ < tokens_.size();
 178 }
 179
 180
 181 char Parser::getChar()
 182 {
 183         if (!good())
 184                 error("The input stream is not well...");
 185         return tokens_[pos_++].character();
 186 }
 187
 188
 189 string Parser::getArg(char left, char right)
 190 {
 191         skipSpaces();
 192
 193         string result;
 194         char c = getChar();
 195
 196         if (c != left)
 197                 putback();
 198         else
 199                 while ((c = getChar()) != right && good())
 200                         result += c;
 201
 202         return result;
 203 }
 204
 205
 206 string Parser::getOpt()
 207 {
 208         string res = getArg('[', ']');
 209         return res.size() ? '[' + res + ']' : string();
 210 }
 211
 212
 213 void Parser::tokenize(istream & is)
 214 {
 215         static bool init_done = false;
 216
 217         if (!init_done) {
 218                 catInit();
 219                 init_done = true;
 220         }
 221
 222         char c;
 223         while (is.get(c)) {
 224                 //cerr << "reading c: " << c << "\n";
 225
 226                 switch (catcode(c)) {
 227                         case catNewline: {
 228                                 ++lineno_;
 229                                 is.get(c);
 230                                 if (catcode(c) == catNewline)
 231                                         push_back(Token("par"));
 232                                 else {
 233                                         push_back(Token('\n', catNewline));
 234                                         is.putback(c);
 235                                 }
 236                                 break;
 237                         }
 238
 239 /*
 240                         case catComment: {
 241                                 while (is.get(c) && catcode(c) != catNewline)
 242                                         ;
 243                                 ++lineno_;
 244                                 break;
 245                         }
 246 */
 247
 248                         case catEscape: {
 249                                 is.get(c);
 250                                 if (!is) {
 251                                         error("unexpected end of input");
 252                                 } else {
 253                                         string s(1, c);
 254                                         if (catcode(c) == catLetter) {
 255                                                 // collect letters
 256                                                 while (is.get(c) && catcode(c) == catLetter)
 257                                                         s += c;
 258                                                 skipSpaceTokens(is, c);
 259                                         }
 260                                         push_back(Token(s));
 261                                 }
 262                                 break;
 263                         }
 264
 265                         case catSuper:
 266                         case catSub: {
 267                                 push_back(Token(c, catcode(c)));
 268                                 is.get(c);
 269                                 skipSpaceTokens(is, c);
 270                                 break;
 271                         }
 272
 273                         case catIgnore: {
 274                                 if (c != 13)
 275                                         cerr << "ignoring a char: " << int(c) << "\n";
 276                                 break;
 277                         }
 278
 279                         default:
 280                                 push_back(Token(c, catcode(c)));
 281                 }
 282         }
 283 }
 284
 285
 286 void Parser::dump() const
 287 {
 288         cerr << "\nTokens: ";
 289         for (unsigned i = 0; i < tokens_.size(); ++i) {
 290                 if (i == pos_)
 291                         cerr << " <#> ";
 292                 cerr << tokens_[i];
 293         }
 294         cerr << " pos: " << pos_ << "\n";
 295 }
 296
 297
 298 void Parser::error(string const & msg)
 299 {
 300         cerr << "Line ~" << lineno_ << ":  parse error: " << msg << endl;
 301         dump();
 302         //exit(1);
 303 }
 304
 305
 306 string Parser::verbatimOption()
 307 {
 308         string res;
 309         if (nextToken().character() == '[') {
 310                 Token t = getToken();
 311                 for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
 312                         if (t.cat() == catBegin) {
 313                                 putback();
 314                                 res += '{' + verbatimItem() + '}';
 315                         } else
 316                                 res += t.asString();
 317                 }
 318         }
 319         return res;
 320 }
 321
 322
 323 string Parser::verbatimItem()
 324 {
 325         if (!good())
 326                 error("stream bad");
 327         skipSpaces();
 328         if (nextToken().cat() == catBegin) {
 329                 Token t = getToken(); // skip brace
 330                 string res;
 331                 for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
 332                         if (t.cat() == catBegin) {
 333                                 putback();
 334                                 res += '{' + verbatimItem() + '}';
 335                         }
 336                         else
 337                                 res += t.asInput();
 338                 }
 339                 return res;
 340         }
 341         return getToken().asInput();
 342 }
 343
 344
 345 void Parser::setCatCode(char c, CatCode cat)
 346 {
 347         theCatcode[c] = cat;
 348 }
 349
 350
 351 CatCode Parser::getCatCode(char c) const
 352 {
 353         return theCatcode[c];
 354 }