src/mathed/math_parser.C

   1 /*
   2  *  File:        math_parser.C
   3  *  Purpose:     Parser for mathed
   4  *  Author:      Alejandro Aguilar Sierra <asierra@servidor.unam.mx>
   5  *  Created:     January 1996
   6  *  Description: Parse LaTeX2e math mode code.
   7  *
   8  *  Dependencies: Xlib, XForms
   9  *
  10  *  Copyright: 1996, Alejandro Aguilar Sierra
  11  *
  12  *   Version: 0.8beta.
  13  *
  14  *   You are free to use and modify this code under the terms of
  15  *   the GNU General Public Licence version 2 or later.
  16  */
  17
  18 /*
  19
  20 If someone desperately needs partial "structures" (such as a few cells of
  21 an array inset or similar) (s)he could uses the following hack as starting
  22 point to write some macros:
  23
  24   \newif\ifcomment
  25   \commentfalse
  26   \ifcomment
  27           \def\makeamptab{\catcode`\&=4\relax}
  28           \def\makeampletter{\catcode`\&=11\relax}
  29     \def\b{\makeampletter\expandafter\makeamptab\bi}
  30     \long\def\bi#1\e{}
  31   \else
  32     \def\b{}\def\e{}
  33   \fi
  34
  35   ...
  36
  37   \[\begin{array}{ccc}
  38    1 & 2\b & 3^2\\
  39    4 & 5\e & 6\\
  40    7 & 8 & 9
  41   \end{array}\]
  42
  43 */
  44
  45
  46 #include <config.h>
  47
  48 #ifdef __GNUG__
  49 #pragma implementation
  50 #endif
  51
  52 #include "math_parser.h"
  53 #include "math_inset.h"
  54 #include "math_arrayinset.h"
  55 #include "math_braceinset.h"
  56 #include "math_casesinset.h"
  57 #include "math_charinset.h"
  58 #include "math_deliminset.h"
  59 #include "math_factory.h"
  60 #include "math_funcinset.h"
  61 #include "math_kerninset.h"
  62 #include "math_macro.h"
  63 #include "math_macrotable.h"
  64 #include "math_macrotemplate.h"
  65 #include "math_hullinset.h"
  66 #include "math_rootinset.h"
  67 #include "math_sqrtinset.h"
  68 #include "math_scriptinset.h"
  69 #include "math_specialcharinset.h"
  70 #include "math_splitinset.h"
  71 #include "math_sqrtinset.h"
  72 #include "math_support.h"
  73
  74 #include "lyxlex.h"
  75 #include "debug.h"
  76
  77 #include "support/lstrings.h"
  78
  79 #include <cctype>
  80 #include <stack>
  81 #include <algorithm>
  82
  83 using std::istream;
  84 using std::ostream;
  85 using std::ios;
  86 using std::endl;
  87 using std::stack;
  88 using std::fill;
  89
  90
  91 namespace {
  92
  93 bool stared(string const & s)
  94 {
  95         unsigned const n = s.size();
  96         return n && s[n - 1] == '*';
  97 }
  98
  99
 100 void add(MathArray & ar, char c, MathTextCodes code)
 101 {
 102         ar.push_back(MathAtom(new MathCharInset(c, code)));
 103 }
 104
 105
 106 // These are TeX's catcodes
 107 enum CatCode {
 108         catEscape,     // 0    backslash
 109         catBegin,      // 1    {
 110         catEnd,        // 2    }
 111         catMath,       // 3    $
 112         catAlign,      // 4    &
 113         catNewline,    // 5    ^^M
 114         catParameter,  // 6    #
 115         catSuper,      // 7    ^
 116         catSub,        // 8    _
 117         catIgnore,     // 9
 118         catSpace,      // 10   space
 119         catLetter,     // 11   a-zA-Z
 120         catOther,      // 12   none of the above
 121         catActive,     // 13   ~
 122         catComment,    // 14   %
 123         catInvalid     // 15   <delete>
 124 };
 125
 126 CatCode theCatcode[256];
 127
 128
 129 inline CatCode catcode(unsigned char c)
 130 {
 131         return theCatcode[c];
 132 }
 133
 134
 135 enum {
 136         FLAG_BRACE_LAST = 1 << 1,  //  last closing brace ends the parsing process
 137         FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
 138         FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
 139         FLAG_BRACK_END  = 1 << 4,  //  next closing bracket ends the parsing process
 140         FLAG_BOX        = 1 << 5,  //  we are in a box
 141         FLAG_ITEM       = 1 << 7,  //  read a (possibly braced token)
 142         FLAG_BLOCK      = 1 << 8,  //  next block ends the parsing process
 143         FLAG_LEAVE      = 1 << 9   //  leave the loop at the end
 144 };
 145
 146
 147 void catInit()
 148 {
 149         fill(theCatcode, theCatcode + 256, catOther);
 150         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
 151         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
 152
 153         theCatcode['\\'] = catEscape;
 154         theCatcode['{']  = catBegin;
 155         theCatcode['}']  = catEnd;
 156         theCatcode['$']  = catMath;
 157         theCatcode['&']  = catAlign;
 158         theCatcode['\n'] = catNewline;
 159         theCatcode['#']  = catParameter;
 160         theCatcode['^']  = catSuper;
 161         theCatcode['_']  = catSub;
 162         theCatcode['\7f'] = catIgnore;
 163         theCatcode[' ']  = catSpace;
 164         theCatcode['\t'] = catSpace;
 165         theCatcode['\r'] = catSpace;
 166         theCatcode['~']  = catActive;
 167         theCatcode['%']  = catComment;
 168 }
 169
 170
 171
 172 //
 173 // Helper class for parsing
 174 //
 175
 176 class Token {
 177 public:
 178         ///
 179         Token() : cs_(), char_(0), cat_(catIgnore) {}
 180         ///
 181         Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
 182         ///
 183         Token(string const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
 184
 185         ///
 186         string const & cs() const { return cs_; }
 187         ///
 188         CatCode cat() const { return cat_; }
 189         ///
 190         char character() const { return char_; }
 191         ///
 192         string asString() const;
 193         ///
 194         bool isCR() const;
 195
 196 private:
 197         ///
 198         string cs_;
 199         ///
 200         char char_;
 201         ///
 202         CatCode cat_;
 203 };
 204
 205 bool Token::isCR() const
 206 {
 207         return cs_ == "\\" || cs_ == "cr" || cs_ == "crcr";
 208 }
 209
 210 string Token::asString() const
 211 {
 212         return cs_.size() ? cs_ : string(1, char_);
 213 }
 214
 215 bool operator==(Token const & s, Token const & t)
 216 {
 217         return s.character() == t.character()
 218                 && s.cat() == t.cat() && s.cs() == t.cs();
 219 }
 220
 221 bool operator!=(Token const & s, Token const & t)
 222 {
 223         return !(s == t);
 224 }
 225
 226 ostream & operator<<(ostream & os, Token const & t)
 227 {
 228         if (t.cs().size())
 229                 os << "\\" << t.cs();
 230         else
 231                 os << "[" << t.character() << "," << t.cat() << "]";
 232         return os;
 233 }
 234
 235
 236 class Parser {
 237
 238 public:
 239         ///
 240         Parser(LyXLex & lex);
 241         ///
 242         Parser(istream & is);
 243
 244         ///
 245         string parse_macro();
 246         ///
 247         bool parse_normal(MathAtom &);
 248         ///
 249         void parse_into(MathArray & array, unsigned flags, MathTextCodes = LM_TC_MIN);
 250         ///
 251         int lineno() const { return lineno_; }
 252         ///
 253         void putback();
 254
 255 private:
 256         ///
 257         string getArg(char lf, char rf);
 258         ///
 259         char getChar();
 260         ///
 261         void error(string const & msg);
 262         ///
 263         bool parse_lines(MathAtom & t, bool numbered, bool outmost);
 264
 265 private:
 266         ///
 267         void tokenize(istream & is);
 268         ///
 269         void tokenize(string const & s);
 270         ///
 271         void push_back(Token const & t);
 272         ///
 273         void pop_back();
 274         ///
 275         Token const & prevToken() const;
 276         ///
 277         Token const & nextToken() const;
 278         ///
 279         Token const & getToken();
 280         /// skips spaces if any
 281         void skipSpaces();
 282         /// counts a sequence of hlines
 283         int readHLines();
 284         ///
 285         void lex(string const & s);
 286         ///
 287         bool good() const;
 288
 289         ///
 290         int lineno_;
 291         ///
 292         std::vector<Token> tokens_;
 293         ///
 294         unsigned pos_;
 295         ///
 296         bool   curr_num_;
 297         ///
 298         string curr_label_;
 299         ///
 300         string curr_skip_;
 301 };
 302
 303
 304 Parser::Parser(LyXLex & lexer)
 305         : lineno_(lexer.getLineNo()), pos_(0), curr_num_(false)
 306 {
 307         tokenize(lexer.getStream());
 308         lexer.eatLine();
 309 }
 310
 311
 312 Parser::Parser(istream & is)
 313         : lineno_(0), pos_(0), curr_num_(false)
 314 {
 315         tokenize(is);
 316 }
 317
 318
 319 void Parser::push_back(Token const & t)
 320 {
 321         tokens_.push_back(t);
 322 }
 323
 324
 325 void Parser::pop_back()
 326 {
 327         tokens_.pop_back();
 328 }
 329
 330
 331 Token const & Parser::prevToken() const
 332 {
 333         static const Token dummy;
 334         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 335 }
 336
 337
 338 Token const & Parser::nextToken() const
 339 {
 340         static const Token dummy;
 341         return good() ? tokens_[pos_] : dummy;
 342 }
 343
 344
 345 Token const & Parser::getToken()
 346 {
 347         static const Token dummy;
 348         //lyxerr << "looking at token " << tokens_[pos_] << '\n';
 349         return good() ? tokens_[pos_++] : dummy;
 350 }
 351
 352
 353 void Parser::skipSpaces()
 354 {
 355         while (nextToken().cat() == catSpace)
 356                 getToken();
 357 }
 358
 359
 360 int Parser::readHLines()
 361 {
 362         int num = 0;
 363         skipSpaces();
 364         while (nextToken().cs() == "hline") {
 365                 getToken();
 366                 ++num;
 367                 skipSpaces();
 368         }
 369         return num;
 370 }
 371
 372
 373 void Parser::putback()
 374 {
 375         --pos_;
 376 }
 377
 378
 379 bool Parser::good() const
 380 {
 381         return pos_ < tokens_.size();
 382 }
 383
 384
 385 char Parser::getChar()
 386 {
 387         if (!good())
 388                 lyxerr << "The input stream is not well..." << endl;
 389         return tokens_[pos_++].character();
 390 }
 391
 392
 393 string Parser::getArg(char lf, char rg)
 394 {
 395         string result;
 396         char c = getChar();
 397
 398         if (c != lf)
 399                 putback();
 400         else
 401                 while ((c = getChar()) != rg && good())
 402                         result += c;
 403
 404         return result;
 405 }
 406
 407
 408 void Parser::tokenize(istream & is)
 409 {
 410         // eat everything up to the next \end_inset or end of stream
 411         // and store it in s for further tokenization
 412         string s;
 413         char c;
 414         while (is.get(c)) {
 415                 s += c;
 416                 if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
 417                         s = s.substr(0, s.size() - 10);
 418                         break;
 419                 }
 420         }
 421
 422         // tokenize buffer
 423         tokenize(s);
 424 }
 425
 426
 427 void Parser::tokenize(string const & buffer)
 428 {
 429         static bool init_done = false;
 430
 431         if (!init_done) {
 432                 catInit();
 433                 init_done = true;
 434         }
 435
 436         istringstream is(buffer.c_str(), ios::in | ios::binary);
 437
 438         char c;
 439         while (is.get(c)) {
 440
 441                 switch (catcode(c)) {
 442                         case catNewline: {
 443                                 ++lineno_;
 444                                 is.get(c);
 445                                 if (catcode(c) == catNewline)
 446                                         ; //push_back(Token("par"));
 447                                 else {
 448                                         push_back(Token(' ', catSpace));
 449                                         is.putback(c);
 450                                 }
 451                                 break;
 452                         }
 453
 454                         case catComment: {
 455                                 while (is.get(c) && catcode(c) != catNewline)
 456                                         ;
 457                                 ++lineno_;
 458                                 break;
 459                         }
 460
 461                         case catEscape: {
 462                                 is.get(c);
 463                                 string s(1, c);
 464                                 if (catcode(c) == catLetter) {
 465                                         while (is.get(c) && catcode(c) == catLetter)
 466                                                 s += c;
 467                                         if (catcode(c) == catSpace)
 468                                                 while (is.get(c) && catcode(c) == catSpace)
 469                                                         ;
 470                                         is.putback(c);
 471                                 }
 472                                 push_back(Token(s));
 473                                 break;
 474                         }
 475
 476                         default:
 477                                 push_back(Token(c, catcode(c)));
 478                 }
 479         }
 480
 481 #if 0
 482         lyxerr << "\nTokens: ";
 483         for (unsigned i = 0; i < tokens_.size(); ++i)
 484                 lyxerr << tokens_[i];
 485         lyxerr << "\n";
 486 #endif
 487 }
 488
 489
 490 void Parser::error(string const & msg)
 491 {
 492         lyxerr << "Line ~" << lineno_ << ": Math parse error: " << msg << endl;
 493         //exit(1);
 494 }
 495
 496
 497 bool Parser::parse_lines(MathAtom & t, bool numbered, bool outmost)
 498 {
 499         MathGridInset * p = t->asGridInset();
 500         if (!p) {
 501                 lyxerr << "error in Parser::parse_lines() 1\n";
 502                 return false;
 503         }
 504
 505         int const cols = p->ncols();
 506
 507         // save global variables
 508         bool   const saved_num   = curr_num_;
 509         string const saved_label = curr_label_;
 510
 511         // read initial hlines
 512         p->rowinfo(0).lines_ = readHLines();
 513
 514         for (int row = 0; true; ++row) {
 515                 // reset global variables
 516                 curr_num_   = numbered;
 517                 curr_label_.erase();
 518
 519                 // reading a row
 520                 for (int col = 0; col < cols; ++col) {
 521                         //lyxerr << "reading cell " << row << " " << col << "\n";
 522                         parse_into(p->cell(col + row * cols), FLAG_BLOCK);
 523
 524                         // break if cell is not followed by an ampersand
 525                         if (nextToken().cat() != catAlign) {
 526                                 //lyxerr << "less cells read than normal in row/col: "
 527                                 //      << row << " " << col << "\n";
 528                                 break;
 529                         }
 530
 531                         // skip the ampersand
 532                         getToken();
 533                 }
 534
 535                 if (outmost) {
 536                         MathHullInset * m = t->asHullInset();
 537                         if (!m) {
 538                                 lyxerr << "error in Parser::parse_lines() 2\n";
 539                                 return false;
 540                         }
 541                         m->numbered(row, curr_num_);
 542                         m->label(row, curr_label_);
 543                         if (curr_skip_.size()) {
 544                                 m->vcrskip(LyXLength(curr_skip_), row);
 545                                 curr_skip_.erase();
 546                         }
 547                 }
 548
 549                 // is a \\ coming?
 550                 if (nextToken().isCR()) {
 551                         // skip the cr-token
 552                         getToken();
 553
 554                         // try to read a length
 555                         //get
 556
 557                         // read hlines for next row
 558                         p->rowinfo(row + 1).lines_ = readHLines();
 559                 }
 560
 561                 // we are finished if the next token is an 'end'
 562                 if (nextToken().cs() == "end") {
 563                         // skip the end-token
 564                         getToken();
 565                         getArg('{','}');
 566
 567                         // leave the 'read a line'-loop
 568                         break;
 569                 }
 570
 571                 // otherwise, we have to start a new row
 572                 p->appendRow();
 573         }
 574
 575         // restore "global" variables
 576         curr_num_   = saved_num;
 577         curr_label_ = saved_label;
 578
 579         return true;
 580 }
 581
 582
 583 string Parser::parse_macro()
 584 {
 585         string name = "{error}";
 586         skipSpaces();
 587
 588         if (getToken().cs() != "newcommand") {
 589                 lyxerr << "\\newcommand expected\n";
 590                 return name;
 591         }
 592
 593         if (getToken().cat() != catBegin) {
 594                 lyxerr << "'{' in \\newcommand expected (1)\n";
 595                 return name;
 596         }
 597
 598         name = getToken().cs();
 599
 600         if (getToken().cat() != catEnd) {
 601                 lyxerr << "'}' expected\n";
 602                 return name;
 603         }
 604
 605         string    arg  = getArg('[', ']');
 606         int       narg = arg.empty() ? 0 : atoi(arg.c_str());
 607
 608         if (getToken().cat() != catBegin) {
 609                 lyxerr << "'{' in \\newcommand expected (2)\n";
 610                 return name;
 611         }
 612
 613         MathArray ar;
 614         parse_into(ar, FLAG_BRACE_LAST);
 615         MathMacroTable::create(name, narg, ar);
 616         return name;
 617 }
 618
 619
 620 bool Parser::parse_normal(MathAtom & matrix)
 621 {
 622         skipSpaces();
 623         Token const & t = getToken();
 624
 625         if (t.cs() == "(") {
 626                 matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 627                 parse_into(matrix->cell(0), 0);
 628                 return true;
 629         }
 630
 631         if (t.cat() == catMath) {
 632                 Token const & n = getToken();
 633                 if (n.cat() == catMath) {
 634                         // TeX's $$...$$ syntax for displayed math
 635                         matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 636                         MathHullInset * p = matrix->asHullInset();
 637                         parse_into(p->cell(0), 0);
 638                         p->numbered(0, curr_num_);
 639                         p->label(0, curr_label_);
 640                 } else {
 641                         // simple $...$  stuff
 642                         putback();
 643                         matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 644                         parse_into(matrix->cell(0), 0);
 645                 }
 646                 return true;
 647         }
 648
 649         if (!t.cs().size()) {
 650                 lyxerr << "start of math expected, got '" << t << "'\n";
 651                 return false;
 652         }
 653
 654         string const & cs = t.cs();
 655
 656         if (cs == "[") {
 657                 curr_num_ = 0;
 658                 curr_label_.erase();
 659                 matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 660                 MathHullInset * p = matrix->asHullInset();
 661                 parse_into(p->cell(0), 0);
 662                 p->numbered(0, curr_num_);
 663                 p->label(0, curr_label_);
 664                 return true;
 665         }
 666
 667         if (cs != "begin") {
 668                 lyxerr << "'begin' of un-simple math expected, got '" << cs << "'\n";
 669                 return false;
 670         }
 671
 672         string const name = getArg('{', '}');
 673
 674         if (name == "equation" || name == "equation*" || name == "displaymath") {
 675                 curr_num_ = (name == "equation");
 676                 curr_label_.erase();
 677                 matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 678                 MathHullInset * p = matrix->asHullInset();
 679                 parse_into(p->cell(0), FLAG_END);
 680                 p->numbered(0, curr_num_);
 681                 p->label(0, curr_label_);
 682                 return true;
 683         }
 684
 685         if (name == "eqnarray" || name == "eqnarray*") {
 686                 matrix = MathAtom(new MathHullInset(LM_OT_EQNARRAY));
 687                 return parse_lines(matrix, !stared(name), true);
 688         }
 689
 690         if (name == "align" || name == "align*") {
 691                 matrix = MathAtom(new MathHullInset(LM_OT_ALIGN));
 692                 return parse_lines(matrix, !stared(name), true);
 693         }
 694
 695         if (name == "alignat" || name == "alignat*") {
 696                 int nc = 2 * atoi(getArg('{', '}').c_str());
 697                 matrix = MathAtom(new MathHullInset(LM_OT_ALIGNAT, nc));
 698                 return parse_lines(matrix, !stared(name), true);
 699         }
 700
 701         if (name == "xalignat" || name == "xalignat*") {
 702                 int nc = 2 * atoi(getArg('{', '}').c_str());
 703                 matrix = MathAtom(new MathHullInset(LM_OT_XALIGNAT, nc));
 704                 return parse_lines(matrix, !stared(name), true);
 705         }
 706
 707         if (name == "xxalignat") {
 708                 int nc = 2 * atoi(getArg('{', '}').c_str());
 709                 matrix = MathAtom(new MathHullInset(LM_OT_XXALIGNAT, nc));
 710                 return parse_lines(matrix, !stared(name), true);
 711         }
 712
 713         if (name == "multline" || name == "multline*") {
 714                 matrix = MathAtom(new MathHullInset(LM_OT_MULTLINE));
 715                 return parse_lines(matrix, !stared(name), true);
 716         }
 717
 718         if (name == "gather" || name == "gather*") {
 719                 matrix = MathAtom(new MathHullInset(LM_OT_GATHER));
 720                 return parse_lines(matrix, !stared(name), true);
 721         }
 722
 723         lyxerr[Debug::MATHED] << "1: unknown math environment: " << name << "\n";
 724         lyxerr << "1: unknown math environment: " << name << "\n";
 725         return false;
 726 }
 727
 728
 729 void Parser::parse_into(MathArray & array, unsigned flags, MathTextCodes code)
 730 {
 731         bool panic  = false;
 732         int  limits = 0;
 733
 734         while (good()) {
 735                 Token const & t = getToken();
 736
 737                 //lyxerr << "t: " << t << " flags: " << flags << "'\n";
 738                 //array.dump(lyxerr);
 739                 //lyxerr << "\n";
 740
 741                 if (flags & FLAG_ITEM) {
 742                         flags &= ~FLAG_ITEM;
 743                         if (t.cat() == catBegin) {
 744                                 // skip the brace and collect everything to the next matching
 745                                 // closing brace
 746                                 flags |= FLAG_BRACE_LAST;
 747                                 continue;
 748                         } else {
 749                                 // handle only this single token, leave the loop if done
 750                                 flags |= FLAG_LEAVE;
 751                         }
 752                 }
 753
 754                 if (flags & FLAG_BLOCK) {
 755                         if (t.cat() == catAlign || t.isCR() || t.cs() == "end") {
 756                                 putback();
 757                                 return;
 758                         }
 759                 }
 760
 761                 //
 762                 // cat codes
 763                 //
 764                 if (t.cat() == catMath) {
 765                         if (flags & FLAG_BOX) {
 766                                 // we are inside an mbox, so opening new math is allowed
 767                                 array.push_back(MathAtom(new MathHullInset(LM_OT_SIMPLE)));
 768                                 parse_into(array.back()->cell(0), 0);
 769                         } else {
 770                                 // otherwise this is the end of the formula
 771                                 break;
 772                         }
 773                 }
 774
 775                 else if (t.cat() == catLetter)
 776                         add(array, t.character(), code);
 777
 778                 else if (t.cat() == catSpace && code == LM_TC_TEXTRM)
 779                         add(array, t.character(), code);
 780
 781                 else if (t.cat() == catParameter) {
 782                         Token const & n = getToken();
 783                         array.push_back(MathAtom(new MathMacroArgument(n.character() - '0')));
 784                 }
 785
 786                 else if (t.cat() == catBegin) {
 787                         MathArray ar;
 788                         parse_into(ar, FLAG_BRACE_LAST);
 789 #ifndef WITH_WARNINGS
 790 #warning this might be wrong in general!
 791 #endif
 792                         // ignore braces around simple items
 793                         if (ar.size() == 1 || (ar.size() == 2 && ar.back()->asScriptInset())) {
 794                                 array.push_back(ar);
 795                         } else {
 796                                 array.push_back(MathAtom(new MathBraceInset));
 797                                 array.back()->cell(0).swap(ar);
 798                         }
 799                 }
 800
 801                 else if (t.cat() == catEnd) {
 802                         if (flags & FLAG_BRACE_LAST)
 803                                 return;
 804                         lyxerr << "found '}' unexpectedly, array: '" << array << "'\n";
 805                         //lyxerr << "found '}' unexpectedly\n";
 806                         add(array, '}', LM_TC_TEX);
 807                 }
 808
 809                 else if (t.cat() == catAlign) {
 810                         lyxerr << "found tab unexpectedly, array: '" << array << "'\n";
 811                         //lyxerr << "found tab unexpectedly\n";
 812                         add(array, '&', LM_TC_TEX);
 813                 }
 814
 815                 else if (t.cat() == catSuper || t.cat() == catSub) {
 816                         bool up = (t.cat() == catSuper);
 817                         MathScriptInset * p = 0;
 818                         if (array.size())
 819                                 p = array.back()->asScriptInset();
 820                         if (!p || p->has(up)) {
 821                                 array.push_back(MathAtom(new MathScriptInset(up)));
 822                                 p = array.back()->asScriptInset();
 823                         }
 824                         p->ensure(up);
 825                         parse_into(p->cell(up), FLAG_ITEM);
 826                         p->limits(limits);
 827                         limits = 0;
 828                 }
 829
 830                 else if (t.character() == ']' && (flags & FLAG_BRACK_END))
 831                         return;
 832
 833                 else if (t.cat() == catOther)
 834                         add(array, t.character(), code);
 835
 836                 //
 837                 // control sequences
 838                 //
 839                 else if (t.cs() == "protect")
 840                         // ignore \\protect, will be re-added during output
 841                         ;
 842
 843                 else if (t.cs() == "end")
 844                         break;
 845
 846                 else if (t.cs() == ")")
 847                         break;
 848
 849                 else if (t.cs() == "]")
 850                         break;
 851
 852                 else if (t.cs() == "\\") {
 853                         curr_skip_ = getArg('[', ']');
 854                         //lyxerr << "found newline unexpectedly, array: '" << array << "'\n";
 855                         lyxerr << "found newline unexpectedly\n";
 856                         array.push_back(createMathInset("\\"));
 857                 }
 858
 859                 else if (t.cs() == "limits")
 860                         limits = 1;
 861
 862                 else if (t.cs() == "nolimits")
 863                         limits = -1;
 864
 865                 else if (t.cs() == "nonumber")
 866                         curr_num_ = false;
 867
 868                 else if (t.cs() == "number")
 869                         curr_num_ = true;
 870
 871                 else if (t.cs() == "sqrt") {
 872                         char c = getChar();
 873                         if (c == '[') {
 874                                 array.push_back(MathAtom(new MathRootInset));
 875                                 parse_into(array.back()->cell(0), FLAG_BRACK_END);
 876                                 parse_into(array.back()->cell(1), FLAG_ITEM);
 877                         } else {
 878                                 putback();
 879                                 array.push_back(MathAtom(new MathSqrtInset));
 880                                 parse_into(array.back()->cell(0), FLAG_ITEM);
 881                         }
 882                 }
 883
 884                 else if (t.cs() == "left") {
 885                         string l = getToken().asString();
 886                         MathArray ar;
 887                         parse_into(ar, FLAG_RIGHT);
 888                         string r = getToken().asString();
 889                         MathAtom dl(new MathDelimInset(l, r));
 890                         dl->cell(0) = ar;
 891                         array.push_back(dl);
 892                 }
 893
 894                 else if (t.cs() == "right") {
 895                         if (!(flags & FLAG_RIGHT)) {
 896                                 //lyxerr << "got so far: '" << array << "'\n";
 897                                 error("Unmatched right delimiter");
 898                         }
 899                         return;
 900                 }
 901
 902 /*
 903                 case LM_TK_STY:
 904                 {
 905                         lyxerr[Debug::MATHED] << "LM_TK_STY not implemented\n";
 906                         //MathArray tmp = array;
 907                         //MathSizeInset * p = new MathSizeInset(MathStyles(lval_->id));
 908                         //array.push_back(p);
 909                         //parse_into(p->cell(0), FLAG_BRACE_FONT);
 910                         break;
 911                 }
 912
 913 */
 914
 915                 else if (t.cs() == "begin") {
 916                         string const name = getArg('{', '}');
 917                         if (name == "array") {
 918                                 string const valign = getArg('[', ']') + 'c';
 919                                 string const halign = getArg('{', '}');
 920                                 array.push_back(MathAtom(new MathArrayInset(valign[0], halign)));
 921                                 parse_lines(array.back(), false, false);
 922                         } else if (name == "split") {
 923                                 array.push_back(MathAtom(new MathSplitInset(1)));
 924                                 parse_lines(array.back(), false, false);
 925                         } else if (name == "cases") {
 926                                 array.push_back(MathAtom(new MathCasesInset));
 927                                 parse_lines(array.back(), false, false);
 928                         } else
 929                                 lyxerr << "unknow math inset begin '" << name << "'\n";
 930                 }
 931
 932                 else if (t.cs() == "kern") {
 933 #ifdef WITH_WARNINGS
 934 #warning A hack...
 935 #endif
 936                         string s;
 937                         while (1) {
 938                                 Token const & t = getToken();
 939                                 if (!good()) {
 940                                         putback();
 941                                         break;
 942                                 }
 943                                 s += t.character();
 944                                 if (isValidLength(s))
 945                                         break;
 946                         }
 947                         array.push_back(MathAtom(new MathKernInset(s)));
 948                 }
 949
 950                 else if (t.cs() == "label") {
 951                         curr_label_ = getArg('{', '}');
 952                 }
 953
 954                 else if (t.cs() == "choose" || t.cs() == "over" || t.cs() == "atop") {
 955                         MathAtom p = createMathInset(t.cs());
 956                         array.swap(p->cell(0));
 957                         parse_into(p->cell(1), flags, code);
 958                         array.push_back(p);
 959                         return;
 960                 }
 961
 962 /*
 963                 // Disabled
 964                 else if (t.cs() == "mbox") {
 965                         array.push_back(createMathInset(t.cs()));
 966                         // slurp in the argument of mbox
 967
 968                         MathBoxInset * p = array.back()->asBoxInset();
 969                         //lyx::assert(p);
 970                 }
 971 */
 972
 973                 else if (t.cs().size()) {
 974                         latexkeys const * l = in_word_set(t.cs());
 975                         if (l) {
 976                                 if (l->token == LM_TK_FONT) {
 977                                         //lyxerr << "starting font\n";
 978                                         //CatCode catSpaceSave = theCatcode[' '];
 979                                         //if (l->id == LM_TC_TEXTRM) {
 980                                         //      // temporarily change catcode
 981                                         //      theCatcode[' '] = catLetter;
 982                                         //}
 983
 984                                         MathArray ar;
 985                                         parse_into(ar, FLAG_ITEM, static_cast<MathTextCodes>(l->id));
 986                                         array.push_back(ar);
 987
 988                                         // undo catcode changes
 989                                         ////theCatcode[' '] = catSpaceSave;
 990                                         //lyxerr << "ending font\n";
 991                                 }
 992
 993                                 else if (l->token == LM_TK_OLDFONT) {
 994                                         code = static_cast<MathTextCodes>(l->id);
 995                                 }
 996
 997                                 else if (l->token == LM_TK_BOX) {
 998                                         MathAtom p = createMathInset(t.cs());
 999                                         parse_into(p->cell(0), FLAG_ITEM | FLAG_BOX, LM_TC_BOX);
1000                                         array.push_back(p);
1001                                 }
1002
1003                                 else {
1004                                         MathAtom p = createMathInset(t.cs());
1005                                         for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
1006                                                 parse_into(p->cell(i), FLAG_ITEM);
1007                                         array.push_back(p);
1008                                 }
1009                         }
1010
1011                         else {
1012                                 MathAtom p = createMathInset(t.cs());
1013                                 for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
1014                                         parse_into(p->cell(i), FLAG_ITEM);
1015                                 array.push_back(p);
1016                         }
1017                 }
1018
1019
1020                 if (flags & FLAG_LEAVE) {
1021                         flags &= ~FLAG_LEAVE;
1022                         break;
1023                 }
1024         }
1025
1026         if (panic) {
1027                 lyxerr << " Math Panic, expect problems!\n";
1028                 //   Search for the end command.
1029                 Token t;
1030                 do {
1031                         t = getToken();
1032                 } while (good() && t.cs() != "end");
1033         }
1034 }
1035
1036
1037
1038 } // anonymous namespace
1039
1040
1041 void mathed_parse_cell(MathArray & ar, string const & str)
1042 {
1043         istringstream is(str.c_str());
1044         mathed_parse_cell(ar, is);
1045 }
1046
1047
1048 void mathed_parse_cell(MathArray & ar, istream & is)
1049 {
1050         Parser(is).parse_into(ar, 0);
1051 }
1052
1053
1054
1055 string mathed_parse_macro(string const & str)
1056 {
1057         istringstream is(str.c_str());
1058         Parser parser(is);
1059         return parser.parse_macro();
1060 }
1061
1062 string mathed_parse_macro(istream & is)
1063 {
1064         Parser parser(is);
1065         return parser.parse_macro();
1066 }
1067
1068 string mathed_parse_macro(LyXLex & lex)
1069 {
1070         Parser parser(lex);
1071         return parser.parse_macro();
1072 }
1073
1074
1075
1076 bool mathed_parse_normal(MathAtom & t, string const & str)
1077 {
1078         istringstream is(str.c_str());
1079         Parser parser(is);
1080         return parser.parse_normal(t);
1081 }
1082
1083 bool mathed_parse_normal(MathAtom & t, istream & is)
1084 {
1085         Parser parser(is);
1086         return parser.parse_normal(t);
1087 }
1088
1089 bool mathed_parse_normal(MathAtom & t, LyXLex & lex)
1090 {
1091         Parser parser(lex);
1092         return parser.parse_normal(t);
1093 }