src/mathed/math_parser.C

   1 /*
   2  *  File:        math_parser.C
   3  *  Purpose:     Parser for mathed
   4  *  Author:      Alejandro Aguilar Sierra <asierra@servidor.unam.mx>
   5  *  Created:     January 1996
   6  *  Description: Parse LaTeX2e math mode code.
   7  *
   8  *  Dependencies: Xlib, XForms
   9  *
  10  *  Copyright: 1996, Alejandro Aguilar Sierra
  11  *
  12  *   Version: 0.8beta.
  13  *
  14  *   You are free to use and modify this code under the terms of
  15  *   the GNU General Public Licence version 2 or later.
  16  */
  17
  18 /*
  19
  20 If someone desperately needs partial "structures" (such as a few cells of
  21 an array inset or similar) (s)he could uses the following hack as starting
  22 point to write some macros:
  23
  24   \newif\ifcomment
  25   \commentfalse
  26   \ifcomment
  27           \def\makeamptab{\catcode`\&=4\relax}
  28           \def\makeampletter{\catcode`\&=11\relax}
  29     \def\b{\makeampletter\expandafter\makeamptab\bi}
  30     \long\def\bi#1\e{}
  31   \else
  32     \def\b{}\def\e{}
  33   \fi
  34
  35   ...
  36
  37   \[\begin{array}{ccc}
  38    1 & 2\b & 3^2\\
  39    4 & 5\e & 6\\
  40    7 & 8 & 9
  41   \end{array}\]
  42
  43 */
  44
  45
  46 #include <config.h>
  47
  48 #ifdef __GNUG__
  49 #pragma implementation
  50 #endif
  51
  52 #include "math_parser.h"
  53 #include "math_inset.h"
  54 #include "math_arrayinset.h"
  55 #include "math_braceinset.h"
  56 #include "math_casesinset.h"
  57 #include "math_charinset.h"
  58 #include "math_deliminset.h"
  59 #include "math_factory.h"
  60 #include "math_funcinset.h"
  61 #include "math_kerninset.h"
  62 #include "math_macro.h"
  63 #include "math_macrotable.h"
  64 #include "math_macrotemplate.h"
  65 #include "math_hullinset.h"
  66 #include "math_rootinset.h"
  67 #include "math_sizeinset.h"
  68 #include "math_sqrtinset.h"
  69 #include "math_scriptinset.h"
  70 #include "math_specialcharinset.h"
  71 #include "math_splitinset.h"
  72 #include "math_sqrtinset.h"
  73 #include "math_support.h"
  74
  75 #include "lyxlex.h"
  76 #include "debug.h"
  77
  78 #include "support/lstrings.h"
  79
  80 #include <cctype>
  81 #include <stack>
  82 #include <algorithm>
  83
  84 using std::istream;
  85 using std::ostream;
  86 using std::ios;
  87 using std::endl;
  88 using std::stack;
  89 using std::fill;
  90
  91
  92 namespace {
  93
  94 bool stared(string const & s)
  95 {
  96         unsigned const n = s.size();
  97         return n && s[n - 1] == '*';
  98 }
  99
 100
 101 void add(MathArray & ar, char c, MathTextCodes code)
 102 {
 103         ar.push_back(MathAtom(new MathCharInset(c, code)));
 104 }
 105
 106
 107 // These are TeX's catcodes
 108 enum CatCode {
 109         catEscape,     // 0    backslash
 110         catBegin,      // 1    {
 111         catEnd,        // 2    }
 112         catMath,       // 3    $
 113         catAlign,      // 4    &
 114         catNewline,    // 5    ^^M
 115         catParameter,  // 6    #
 116         catSuper,      // 7    ^
 117         catSub,        // 8    _
 118         catIgnore,     // 9
 119         catSpace,      // 10   space
 120         catLetter,     // 11   a-zA-Z
 121         catOther,      // 12   none of the above
 122         catActive,     // 13   ~
 123         catComment,    // 14   %
 124         catInvalid     // 15   <delete>
 125 };
 126
 127 CatCode theCatcode[256];
 128
 129
 130 inline CatCode catcode(unsigned char c)
 131 {
 132         return theCatcode[c];
 133 }
 134
 135
 136 enum {
 137         FLAG_BRACE_LAST = 1 << 1,  //  last closing brace ends the parsing process
 138         FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
 139         FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
 140         FLAG_BRACK_END  = 1 << 4,  //  next closing bracket ends the parsing process
 141         FLAG_BOX        = 1 << 5,  //  we are in a box
 142         FLAG_ITEM       = 1 << 7,  //  read a (possibly braced token)
 143         FLAG_BLOCK      = 1 << 8,  //  next block ends the parsing process
 144         FLAG_LEAVE      = 1 << 9   //  leave the loop at the end
 145 };
 146
 147
 148 void catInit()
 149 {
 150         fill(theCatcode, theCatcode + 256, catOther);
 151         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
 152         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
 153
 154         theCatcode['\\'] = catEscape;
 155         theCatcode['{']  = catBegin;
 156         theCatcode['}']  = catEnd;
 157         theCatcode['$']  = catMath;
 158         theCatcode['&']  = catAlign;
 159         theCatcode['\n'] = catNewline;
 160         theCatcode['#']  = catParameter;
 161         theCatcode['^']  = catSuper;
 162         theCatcode['_']  = catSub;
 163         theCatcode['\7f'] = catIgnore;
 164         theCatcode[' ']  = catSpace;
 165         theCatcode['\t'] = catSpace;
 166         theCatcode['\r'] = catSpace;
 167         theCatcode['~']  = catActive;
 168         theCatcode['%']  = catComment;
 169 }
 170
 171
 172
 173 //
 174 // Helper class for parsing
 175 //
 176
 177 class Token {
 178 public:
 179         ///
 180         Token() : cs_(), char_(0), cat_(catIgnore) {}
 181         ///
 182         Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
 183         ///
 184         Token(string const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
 185
 186         ///
 187         string const & cs() const { return cs_; }
 188         ///
 189         CatCode cat() const { return cat_; }
 190         ///
 191         char character() const { return char_; }
 192         ///
 193         string asString() const;
 194         ///
 195         bool isCR() const;
 196
 197 private:
 198         ///
 199         string cs_;
 200         ///
 201         char char_;
 202         ///
 203         CatCode cat_;
 204 };
 205
 206 bool Token::isCR() const
 207 {
 208         return cs_ == "\\" || cs_ == "cr" || cs_ == "crcr";
 209 }
 210
 211 string Token::asString() const
 212 {
 213         return cs_.size() ? cs_ : string(1, char_);
 214 }
 215
 216 bool operator==(Token const & s, Token const & t)
 217 {
 218         return s.character() == t.character()
 219                 && s.cat() == t.cat() && s.cs() == t.cs();
 220 }
 221
 222 bool operator!=(Token const & s, Token const & t)
 223 {
 224         return !(s == t);
 225 }
 226
 227 ostream & operator<<(ostream & os, Token const & t)
 228 {
 229         if (t.cs().size())
 230                 os << "\\" << t.cs();
 231         else
 232                 os << "[" << t.character() << "," << t.cat() << "]";
 233         return os;
 234 }
 235
 236
 237 class Parser {
 238
 239 public:
 240         ///
 241         Parser(LyXLex & lex);
 242         ///
 243         Parser(istream & is);
 244
 245         ///
 246         string parse_macro();
 247         ///
 248         bool parse_normal(MathAtom &);
 249         ///
 250         void parse_into(MathArray & array, unsigned flags, MathTextCodes = LM_TC_MIN);
 251         ///
 252         int lineno() const { return lineno_; }
 253         ///
 254         void putback();
 255
 256 private:
 257         ///
 258         string getArg(char lf, char rf);
 259         ///
 260         char getChar();
 261         ///
 262         void error(string const & msg);
 263         ///
 264         bool parse_lines(MathAtom & t, bool numbered, bool outmost);
 265
 266 private:
 267         ///
 268         void tokenize(istream & is);
 269         ///
 270         void tokenize(string const & s);
 271         ///
 272         void push_back(Token const & t);
 273         ///
 274         void pop_back();
 275         ///
 276         Token const & prevToken() const;
 277         ///
 278         Token const & nextToken() const;
 279         ///
 280         Token const & getToken();
 281         /// skips spaces if any
 282         void skipSpaces();
 283         /// counts a sequence of hlines
 284         int readHLines();
 285         ///
 286         void lex(string const & s);
 287         ///
 288         bool good() const;
 289
 290         ///
 291         int lineno_;
 292         ///
 293         std::vector<Token> tokens_;
 294         ///
 295         unsigned pos_;
 296         ///
 297         bool   curr_num_;
 298         ///
 299         string curr_label_;
 300         ///
 301         string curr_skip_;
 302 };
 303
 304
 305 Parser::Parser(LyXLex & lexer)
 306         : lineno_(lexer.getLineNo()), pos_(0), curr_num_(false)
 307 {
 308         tokenize(lexer.getStream());
 309         lexer.eatLine();
 310 }
 311
 312
 313 Parser::Parser(istream & is)
 314         : lineno_(0), pos_(0), curr_num_(false)
 315 {
 316         tokenize(is);
 317 }
 318
 319
 320 void Parser::push_back(Token const & t)
 321 {
 322         tokens_.push_back(t);
 323 }
 324
 325
 326 void Parser::pop_back()
 327 {
 328         tokens_.pop_back();
 329 }
 330
 331
 332 Token const & Parser::prevToken() const
 333 {
 334         static const Token dummy;
 335         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 336 }
 337
 338
 339 Token const & Parser::nextToken() const
 340 {
 341         static const Token dummy;
 342         return good() ? tokens_[pos_] : dummy;
 343 }
 344
 345
 346 Token const & Parser::getToken()
 347 {
 348         static const Token dummy;
 349         //lyxerr << "looking at token " << tokens_[pos_] << '\n';
 350         return good() ? tokens_[pos_++] : dummy;
 351 }
 352
 353
 354 void Parser::skipSpaces()
 355 {
 356         while (nextToken().cat() == catSpace)
 357                 getToken();
 358 }
 359
 360
 361 int Parser::readHLines()
 362 {
 363         int num = 0;
 364         skipSpaces();
 365         while (nextToken().cs() == "hline") {
 366                 getToken();
 367                 ++num;
 368                 skipSpaces();
 369         }
 370         return num;
 371 }
 372
 373
 374 void Parser::putback()
 375 {
 376         --pos_;
 377 }
 378
 379
 380 bool Parser::good() const
 381 {
 382         return pos_ < tokens_.size();
 383 }
 384
 385
 386 char Parser::getChar()
 387 {
 388         if (!good())
 389                 lyxerr << "The input stream is not well..." << endl;
 390         return tokens_[pos_++].character();
 391 }
 392
 393
 394 string Parser::getArg(char lf, char rg)
 395 {
 396         string result;
 397         char c = getChar();
 398
 399         if (c != lf)
 400                 putback();
 401         else
 402                 while ((c = getChar()) != rg && good())
 403                         result += c;
 404
 405         return result;
 406 }
 407
 408
 409 void Parser::tokenize(istream & is)
 410 {
 411         // eat everything up to the next \end_inset or end of stream
 412         // and store it in s for further tokenization
 413         string s;
 414         char c;
 415         while (is.get(c)) {
 416                 s += c;
 417                 if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
 418                         s = s.substr(0, s.size() - 10);
 419                         break;
 420                 }
 421         }
 422
 423         // tokenize buffer
 424         tokenize(s);
 425 }
 426
 427
 428 void Parser::tokenize(string const & buffer)
 429 {
 430         static bool init_done = false;
 431
 432         if (!init_done) {
 433                 catInit();
 434                 init_done = true;
 435         }
 436
 437         istringstream is(buffer.c_str(), ios::in | ios::binary);
 438
 439         char c;
 440         while (is.get(c)) {
 441
 442                 switch (catcode(c)) {
 443                         case catNewline: {
 444                                 ++lineno_;
 445                                 is.get(c);
 446                                 if (catcode(c) == catNewline)
 447                                         ; //push_back(Token("par"));
 448                                 else {
 449                                         push_back(Token(' ', catSpace));
 450                                         is.putback(c);
 451                                 }
 452                                 break;
 453                         }
 454
 455                         case catComment: {
 456                                 while (is.get(c) && catcode(c) != catNewline)
 457                                         ;
 458                                 ++lineno_;
 459                                 break;
 460                         }
 461
 462                         case catEscape: {
 463                                 is.get(c);
 464                                 string s(1, c);
 465                                 if (catcode(c) == catLetter) {
 466                                         while (is.get(c) && catcode(c) == catLetter)
 467                                                 s += c;
 468                                         if (catcode(c) == catSpace)
 469                                                 while (is.get(c) && catcode(c) == catSpace)
 470                                                         ;
 471                                         is.putback(c);
 472                                 }
 473                                 push_back(Token(s));
 474                                 break;
 475                         }
 476
 477                         default:
 478                                 push_back(Token(c, catcode(c)));
 479                 }
 480         }
 481
 482 #if 0
 483         lyxerr << "\nTokens: ";
 484         for (unsigned i = 0; i < tokens_.size(); ++i)
 485                 lyxerr << tokens_[i];
 486         lyxerr << "\n";
 487 #endif
 488 }
 489
 490
 491 void Parser::error(string const & msg)
 492 {
 493         lyxerr << "Line ~" << lineno_ << ": Math parse error: " << msg << endl;
 494         //exit(1);
 495 }
 496
 497
 498 bool Parser::parse_lines(MathAtom & t, bool numbered, bool outmost)
 499 {
 500         MathGridInset * p = t->asGridInset();
 501         if (!p) {
 502                 lyxerr << "error in Parser::parse_lines() 1\n";
 503                 return false;
 504         }
 505
 506         int const cols = p->ncols();
 507
 508         // save global variables
 509         bool   const saved_num   = curr_num_;
 510         string const saved_label = curr_label_;
 511
 512         // read initial hlines
 513         p->rowinfo(0).lines_ = readHLines();
 514
 515         for (int row = 0; true; ++row) {
 516                 // reset global variables
 517                 curr_num_   = numbered;
 518                 curr_label_.erase();
 519
 520                 // reading a row
 521                 for (int col = 0; col < cols; ++col) {
 522                         //lyxerr << "reading cell " << row << " " << col << "\n";
 523                         parse_into(p->cell(col + row * cols), FLAG_BLOCK);
 524
 525                         // break if cell is not followed by an ampersand
 526                         if (nextToken().cat() != catAlign) {
 527                                 //lyxerr << "less cells read than normal in row/col: "
 528                                 //      << row << " " << col << "\n";
 529                                 break;
 530                         }
 531
 532                         // skip the ampersand
 533                         getToken();
 534                 }
 535
 536                 if (outmost) {
 537                         MathHullInset * m = t->asHullInset();
 538                         if (!m) {
 539                                 lyxerr << "error in Parser::parse_lines() 2\n";
 540                                 return false;
 541                         }
 542                         m->numbered(row, curr_num_);
 543                         m->label(row, curr_label_);
 544                         if (curr_skip_.size()) {
 545                                 m->vcrskip(LyXLength(curr_skip_), row);
 546                                 curr_skip_.erase();
 547                         }
 548                 }
 549
 550                 // is a \\ coming?
 551                 if (nextToken().isCR()) {
 552                         // skip the cr-token
 553                         getToken();
 554
 555                         // try to read a length
 556                         //get
 557
 558                         // read hlines for next row
 559                         p->rowinfo(row + 1).lines_ = readHLines();
 560                 }
 561
 562                 // we are finished if the next token is an 'end'
 563                 if (nextToken().cs() == "end") {
 564                         // skip the end-token
 565                         getToken();
 566                         getArg('{','}');
 567
 568                         // leave the 'read a line'-loop
 569                         break;
 570                 }
 571
 572                 // otherwise, we have to start a new row
 573                 p->appendRow();
 574         }
 575
 576         // restore "global" variables
 577         curr_num_   = saved_num;
 578         curr_label_ = saved_label;
 579
 580         return true;
 581 }
 582
 583
 584 string Parser::parse_macro()
 585 {
 586         string name = "{error}";
 587         skipSpaces();
 588
 589         if (getToken().cs() != "newcommand") {
 590                 lyxerr << "\\newcommand expected\n";
 591                 return name;
 592         }
 593
 594         if (getToken().cat() != catBegin) {
 595                 lyxerr << "'{' in \\newcommand expected (1)\n";
 596                 return name;
 597         }
 598
 599         name = getToken().cs();
 600
 601         if (getToken().cat() != catEnd) {
 602                 lyxerr << "'}' expected\n";
 603                 return name;
 604         }
 605
 606         string    arg  = getArg('[', ']');
 607         int       narg = arg.empty() ? 0 : atoi(arg.c_str());
 608
 609         if (getToken().cat() != catBegin) {
 610                 lyxerr << "'{' in \\newcommand expected (2)\n";
 611                 return name;
 612         }
 613
 614         MathArray ar;
 615         parse_into(ar, FLAG_BRACE_LAST);
 616         MathMacroTable::create(name, narg, ar);
 617         return name;
 618 }
 619
 620
 621 bool Parser::parse_normal(MathAtom & matrix)
 622 {
 623         skipSpaces();
 624         Token const & t = getToken();
 625
 626         if (t.cs() == "(") {
 627                 matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 628                 parse_into(matrix->cell(0), 0);
 629                 return true;
 630         }
 631
 632         if (t.cat() == catMath) {
 633                 Token const & n = getToken();
 634                 if (n.cat() == catMath) {
 635                         // TeX's $$...$$ syntax for displayed math
 636                         matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 637                         MathHullInset * p = matrix->asHullInset();
 638                         parse_into(p->cell(0), 0);
 639                         p->numbered(0, curr_num_);
 640                         p->label(0, curr_label_);
 641                 } else {
 642                         // simple $...$  stuff
 643                         putback();
 644                         matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 645                         parse_into(matrix->cell(0), 0);
 646                 }
 647                 return true;
 648         }
 649
 650         if (!t.cs().size()) {
 651                 lyxerr << "start of math expected, got '" << t << "'\n";
 652                 return false;
 653         }
 654
 655         string const & cs = t.cs();
 656
 657         if (cs == "[") {
 658                 curr_num_ = 0;
 659                 curr_label_.erase();
 660                 matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 661                 MathHullInset * p = matrix->asHullInset();
 662                 parse_into(p->cell(0), 0);
 663                 p->numbered(0, curr_num_);
 664                 p->label(0, curr_label_);
 665                 return true;
 666         }
 667
 668         if (cs != "begin") {
 669                 lyxerr << "'begin' of un-simple math expected, got '" << cs << "'\n";
 670                 return false;
 671         }
 672
 673         string const name = getArg('{', '}');
 674
 675         if (name == "equation" || name == "equation*" || name == "displaymath") {
 676                 curr_num_ = (name == "equation");
 677                 curr_label_.erase();
 678                 matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 679                 MathHullInset * p = matrix->asHullInset();
 680                 parse_into(p->cell(0), FLAG_END);
 681                 p->numbered(0, curr_num_);
 682                 p->label(0, curr_label_);
 683                 return true;
 684         }
 685
 686         if (name == "eqnarray" || name == "eqnarray*") {
 687                 matrix = MathAtom(new MathHullInset(LM_OT_EQNARRAY));
 688                 return parse_lines(matrix, !stared(name), true);
 689         }
 690
 691         if (name == "align" || name == "align*") {
 692                 matrix = MathAtom(new MathHullInset(LM_OT_ALIGN));
 693                 return parse_lines(matrix, !stared(name), true);
 694         }
 695
 696         if (name == "alignat" || name == "alignat*") {
 697                 int nc = 2 * atoi(getArg('{', '}').c_str());
 698                 matrix = MathAtom(new MathHullInset(LM_OT_ALIGNAT, nc));
 699                 return parse_lines(matrix, !stared(name), true);
 700         }
 701
 702         if (name == "xalignat" || name == "xalignat*") {
 703                 int nc = 2 * atoi(getArg('{', '}').c_str());
 704                 matrix = MathAtom(new MathHullInset(LM_OT_XALIGNAT, nc));
 705                 return parse_lines(matrix, !stared(name), true);
 706         }
 707
 708         if (name == "xxalignat") {
 709                 int nc = 2 * atoi(getArg('{', '}').c_str());
 710                 matrix = MathAtom(new MathHullInset(LM_OT_XXALIGNAT, nc));
 711                 return parse_lines(matrix, !stared(name), true);
 712         }
 713
 714         if (name == "multline" || name == "multline*") {
 715                 matrix = MathAtom(new MathHullInset(LM_OT_MULTLINE));
 716                 return parse_lines(matrix, !stared(name), true);
 717         }
 718
 719         if (name == "gather" || name == "gather*") {
 720                 matrix = MathAtom(new MathHullInset(LM_OT_GATHER));
 721                 return parse_lines(matrix, !stared(name), true);
 722         }
 723
 724         lyxerr[Debug::MATHED] << "1: unknown math environment: " << name << "\n";
 725         lyxerr << "1: unknown math environment: " << name << "\n";
 726         return false;
 727 }
 728
 729
 730 void Parser::parse_into(MathArray & array, unsigned flags, MathTextCodes code)
 731 {
 732         bool panic  = false;
 733         int  limits = 0;
 734
 735         while (good()) {
 736                 Token const & t = getToken();
 737
 738                 //lyxerr << "t: " << t << " flags: " << flags << "'\n";
 739                 //array.dump(lyxerr);
 740                 //lyxerr << "\n";
 741
 742                 if (flags & FLAG_ITEM) {
 743                         flags &= ~FLAG_ITEM;
 744                         if (t.cat() == catBegin) {
 745                                 // skip the brace and collect everything to the next matching
 746                                 // closing brace
 747                                 flags |= FLAG_BRACE_LAST;
 748                                 continue;
 749                         } else {
 750                                 // handle only this single token, leave the loop if done
 751                                 flags |= FLAG_LEAVE;
 752                         }
 753                 }
 754
 755                 if (flags & FLAG_BLOCK) {
 756                         if (t.cat() == catAlign || t.isCR() || t.cs() == "end") {
 757                                 putback();
 758                                 return;
 759                         }
 760                 }
 761
 762                 //
 763                 // cat codes
 764                 //
 765                 if (t.cat() == catMath) {
 766                         if (flags & FLAG_BOX) {
 767                                 // we are inside an mbox, so opening new math is allowed
 768                                 array.push_back(MathAtom(new MathHullInset(LM_OT_SIMPLE)));
 769                                 parse_into(array.back()->cell(0), 0);
 770                         } else {
 771                                 // otherwise this is the end of the formula
 772                                 break;
 773                         }
 774                 }
 775
 776                 else if (t.cat() == catLetter)
 777                         add(array, t.character(), code);
 778
 779                 else if (t.cat() == catSpace && code == LM_TC_TEXTRM)
 780                         add(array, t.character(), code);
 781
 782                 else if (t.cat() == catParameter) {
 783                         Token const & n = getToken();
 784                         array.push_back(MathAtom(new MathMacroArgument(n.character() - '0')));
 785                 }
 786
 787                 else if (t.cat() == catBegin) {
 788                         MathArray ar;
 789                         parse_into(ar, FLAG_BRACE_LAST);
 790 #ifndef WITH_WARNINGS
 791 #warning this might be wrong in general!
 792 #endif
 793                         // ignore braces around simple items
 794                         if (ar.size() == 1 || (ar.size() == 2 && ar.back()->asScriptInset())) {
 795                                 array.push_back(ar);
 796                         } else {
 797                                 array.push_back(MathAtom(new MathBraceInset));
 798                                 array.back()->cell(0).swap(ar);
 799                         }
 800                 }
 801
 802                 else if (t.cat() == catEnd) {
 803                         if (flags & FLAG_BRACE_LAST)
 804                                 return;
 805                         lyxerr << "found '}' unexpectedly, array: '" << array << "'\n";
 806                         //lyxerr << "found '}' unexpectedly\n";
 807                         add(array, '}', LM_TC_TEX);
 808                 }
 809
 810                 else if (t.cat() == catAlign) {
 811                         lyxerr << "found tab unexpectedly, array: '" << array << "'\n";
 812                         //lyxerr << "found tab unexpectedly\n";
 813                         add(array, '&', LM_TC_TEX);
 814                 }
 815
 816                 else if (t.cat() == catSuper || t.cat() == catSub) {
 817                         bool up = (t.cat() == catSuper);
 818                         MathScriptInset * p = 0;
 819                         if (array.size())
 820                                 p = array.back()->asScriptInset();
 821                         if (!p || p->has(up)) {
 822                                 array.push_back(MathAtom(new MathScriptInset(up)));
 823                                 p = array.back()->asScriptInset();
 824                         }
 825                         p->ensure(up);
 826                         parse_into(p->cell(up), FLAG_ITEM);
 827                         p->limits(limits);
 828                         limits = 0;
 829                 }
 830
 831                 else if (t.character() == ']' && (flags & FLAG_BRACK_END))
 832                         return;
 833
 834                 else if (t.cat() == catOther)
 835                         add(array, t.character(), code);
 836
 837                 //
 838                 // control sequences
 839                 //
 840                 else if (t.cs() == "protect")
 841                         // ignore \\protect, will be re-added during output
 842                         ;
 843
 844                 else if (t.cs() == "end")
 845                         break;
 846
 847                 else if (t.cs() == ")")
 848                         break;
 849
 850                 else if (t.cs() == "]")
 851                         break;
 852
 853                 else if (t.cs() == "\\") {
 854                         curr_skip_ = getArg('[', ']');
 855                         //lyxerr << "found newline unexpectedly, array: '" << array << "'\n";
 856                         lyxerr << "found newline unexpectedly\n";
 857                         array.push_back(createMathInset("\\"));
 858                 }
 859
 860                 else if (t.cs() == "limits")
 861                         limits = 1;
 862
 863                 else if (t.cs() == "nolimits")
 864                         limits = -1;
 865
 866                 else if (t.cs() == "nonumber")
 867                         curr_num_ = false;
 868
 869                 else if (t.cs() == "number")
 870                         curr_num_ = true;
 871
 872                 else if (t.cs() == "sqrt") {
 873                         char c = getChar();
 874                         if (c == '[') {
 875                                 array.push_back(MathAtom(new MathRootInset));
 876                                 parse_into(array.back()->cell(0), FLAG_BRACK_END);
 877                                 parse_into(array.back()->cell(1), FLAG_ITEM);
 878                         } else {
 879                                 putback();
 880                                 array.push_back(MathAtom(new MathSqrtInset));
 881                                 parse_into(array.back()->cell(0), FLAG_ITEM);
 882                         }
 883                 }
 884
 885                 else if (t.cs() == "left") {
 886                         string l = getToken().asString();
 887                         MathArray ar;
 888                         parse_into(ar, FLAG_RIGHT);
 889                         string r = getToken().asString();
 890                         MathAtom dl(new MathDelimInset(l, r));
 891                         dl->cell(0) = ar;
 892                         array.push_back(dl);
 893                 }
 894
 895                 else if (t.cs() == "right") {
 896                         if (!(flags & FLAG_RIGHT)) {
 897                                 //lyxerr << "got so far: '" << array << "'\n";
 898                                 error("Unmatched right delimiter");
 899                         }
 900                         return;
 901                 }
 902
 903                 else if (t.cs() == "begin") {
 904                         string const name = getArg('{', '}');
 905                         if (name == "array") {
 906                                 string const valign = getArg('[', ']') + 'c';
 907                                 string const halign = getArg('{', '}');
 908                                 array.push_back(MathAtom(new MathArrayInset(valign[0], halign)));
 909                                 parse_lines(array.back(), false, false);
 910                         } else if (name == "split") {
 911                                 array.push_back(MathAtom(new MathSplitInset(1)));
 912                                 parse_lines(array.back(), false, false);
 913                         } else if (name == "cases") {
 914                                 array.push_back(MathAtom(new MathCasesInset));
 915                                 parse_lines(array.back(), false, false);
 916                         } else
 917                                 lyxerr << "unknow math inset begin '" << name << "'\n";
 918                 }
 919
 920                 else if (t.cs() == "kern") {
 921 #ifdef WITH_WARNINGS
 922 #warning A hack...
 923 #endif
 924                         string s;
 925                         while (1) {
 926                                 Token const & t = getToken();
 927                                 if (!good()) {
 928                                         putback();
 929                                         break;
 930                                 }
 931                                 s += t.character();
 932                                 if (isValidLength(s))
 933                                         break;
 934                         }
 935                         array.push_back(MathAtom(new MathKernInset(s)));
 936                 }
 937
 938                 else if (t.cs() == "label") {
 939                         curr_label_ = getArg('{', '}');
 940                 }
 941
 942                 else if (t.cs() == "choose" || t.cs() == "over" || t.cs() == "atop") {
 943                         MathAtom p = createMathInset(t.cs());
 944                         array.swap(p->cell(0));
 945                         parse_into(p->cell(1), flags, code);
 946                         array.push_back(p);
 947                         return;
 948                 }
 949
 950 /*
 951                 // Disabled
 952                 else if (t.cs() == "mbox") {
 953                         array.push_back(createMathInset(t.cs()));
 954                         // slurp in the argument of mbox
 955
 956                         MathBoxInset * p = array.back()->asBoxInset();
 957                         //lyx::assert(p);
 958                 }
 959 */
 960
 961                 else if (t.cs().size()) {
 962                         latexkeys const * l = in_word_set(t.cs());
 963                         if (l) {
 964                                 if (l->token == LM_TK_FONT) {
 965                                         //lyxerr << "starting font\n";
 966                                         //CatCode catSpaceSave = theCatcode[' '];
 967                                         //if (l->id == LM_TC_TEXTRM) {
 968                                         //      // temporarily change catcode
 969                                         //      theCatcode[' '] = catLetter;
 970                                         //}
 971
 972                                         MathArray ar;
 973                                         parse_into(ar, FLAG_ITEM, static_cast<MathTextCodes>(l->id));
 974                                         array.push_back(ar);
 975
 976                                         // undo catcode changes
 977                                         ////theCatcode[' '] = catSpaceSave;
 978                                         //lyxerr << "ending font\n";
 979                                 }
 980
 981                                 else if (l->token == LM_TK_OLDFONT) {
 982                                         code = static_cast<MathTextCodes>(l->id);
 983                                 }
 984
 985                                 else if (l->token == LM_TK_BOX) {
 986                                         MathAtom p = createMathInset(t.cs());
 987                                         parse_into(p->cell(0), FLAG_ITEM | FLAG_BOX, LM_TC_BOX);
 988                                         array.push_back(p);
 989                                 }
 990
 991                                 else if (l->token == LM_TK_STY) {
 992                                         MathAtom p = createMathInset(t.cs());
 993                                         parse_into(p->cell(0), flags, code);
 994                                         array.push_back(p);
 995                                         return;
 996                                 }
 997
 998                                 else {
 999                                         MathAtom p = createMathInset(t.cs());
1000                                         for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
1001                                                 parse_into(p->cell(i), FLAG_ITEM);
1002                                         array.push_back(p);
1003                                 }
1004                         }
1005
1006                         else {
1007                                 MathAtom p = createMathInset(t.cs());
1008                                 for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
1009                                         parse_into(p->cell(i), FLAG_ITEM);
1010                                 array.push_back(p);
1011                         }
1012                 }
1013
1014
1015                 if (flags & FLAG_LEAVE) {
1016                         flags &= ~FLAG_LEAVE;
1017                         break;
1018                 }
1019         }
1020
1021         if (panic) {
1022                 lyxerr << " Math Panic, expect problems!\n";
1023                 //   Search for the end command.
1024                 Token t;
1025                 do {
1026                         t = getToken();
1027                 } while (good() && t.cs() != "end");
1028         }
1029 }
1030
1031
1032
1033 } // anonymous namespace
1034
1035
1036 void mathed_parse_cell(MathArray & ar, string const & str)
1037 {
1038         istringstream is(str.c_str());
1039         mathed_parse_cell(ar, is);
1040 }
1041
1042
1043 void mathed_parse_cell(MathArray & ar, istream & is)
1044 {
1045         Parser(is).parse_into(ar, 0);
1046 }
1047
1048
1049
1050 string mathed_parse_macro(string const & str)
1051 {
1052         istringstream is(str.c_str());
1053         Parser parser(is);
1054         return parser.parse_macro();
1055 }
1056
1057 string mathed_parse_macro(istream & is)
1058 {
1059         Parser parser(is);
1060         return parser.parse_macro();
1061 }
1062
1063 string mathed_parse_macro(LyXLex & lex)
1064 {
1065         Parser parser(lex);
1066         return parser.parse_macro();
1067 }
1068
1069
1070
1071 bool mathed_parse_normal(MathAtom & t, string const & str)
1072 {
1073         istringstream is(str.c_str());
1074         Parser parser(is);
1075         return parser.parse_normal(t);
1076 }
1077
1078 bool mathed_parse_normal(MathAtom & t, istream & is)
1079 {
1080         Parser parser(is);
1081         return parser.parse_normal(t);
1082 }
1083
1084 bool mathed_parse_normal(MathAtom & t, LyXLex & lex)
1085 {
1086         Parser parser(lex);
1087         return parser.parse_normal(t);
1088 }