src/mathed/math_parser.C

   1 /*
   2  *  File:        math_parser.C
   3  *  Purpose:     Parser for mathed
   4  *  Author:      Alejandro Aguilar Sierra <asierra@servidor.unam.mx>
   5  *  Created:     January 1996
   6  *  Description: Parse LaTeX2e math mode code.
   7  *
   8  *  Dependencies: Xlib, XForms
   9  *
  10  *  Copyright: 1996, Alejandro Aguilar Sierra
  11  *
  12  *   Version: 0.8beta.
  13  *
  14  *   You are free to use and modify this code under the terms of
  15  *   the GNU General Public Licence version 2 or later.
  16  */
  17
  18 /*
  19
  20 If someone desperately needs partial "structures" (such as a few cells of
  21 an array inset or similar) (s)he could uses the following hack as starting
  22 point to write some macros:
  23
  24   \newif\ifcomment
  25   \commentfalse
  26   \ifcomment
  27           \def\makeamptab{\catcode`\&=4\relax}
  28           \def\makeampletter{\catcode`\&=11\relax}
  29     \def\b{\makeampletter\expandafter\makeamptab\bi}
  30     \long\def\bi#1\e{}
  31   \else
  32     \def\b{}\def\e{}
  33   \fi
  34
  35   ...
  36
  37   \[\begin{array}{ccc}
  38    1 & 2\b & 3^2\\
  39    4 & 5\e & 6\\
  40    7 & 8 & 9
  41   \end{array}\]
  42
  43 */
  44
  45
  46 #include <config.h>
  47
  48 #ifdef __GNUG__
  49 #pragma implementation
  50 #endif
  51
  52 #include "math_parser.h"
  53 #include "math_inset.h"
  54 #include "math_arrayinset.h"
  55 #include "math_braceinset.h"
  56 #include "math_boxinset.h"
  57 #include "math_charinset.h"
  58 #include "math_deliminset.h"
  59 #include "math_factory.h"
  60 #include "math_funcinset.h"
  61 #include "math_kerninset.h"
  62 #include "math_macro.h"
  63 #include "math_macrotable.h"
  64 #include "math_macrotemplate.h"
  65 #include "math_hullinset.h"
  66 #include "math_rootinset.h"
  67 #include "math_sizeinset.h"
  68 #include "math_sqrtinset.h"
  69 #include "math_scriptinset.h"
  70 #include "math_specialcharinset.h"
  71 #include "math_sqrtinset.h"
  72 #include "math_support.h"
  73 #include "math_xyarrowinset.h"
  74
  75 #include "lyxlex.h"
  76 #include "debug.h"
  77 #include "support/LAssert.h"
  78 #include "support/lstrings.h"
  79
  80 #include <cctype>
  81 #include <stack>
  82 #include <algorithm>
  83
  84 using std::istream;
  85 using std::ostream;
  86 using std::ios;
  87 using std::endl;
  88 using std::stack;
  89 using std::fill;
  90 using std::vector;
  91 using std::atoi;
  92
  93 //#define FILEDEBUG
  94
  95
  96 namespace {
  97
  98 bool stared(string const & s)
  99 {
 100         string::size_type const n = s.size();
 101         return n && s[n - 1] == '*';
 102 }
 103
 104
 105 void add(MathArray & ar, char c, MathTextCodes code)
 106 {
 107         ar.push_back(MathAtom(new MathCharInset(c, code)));
 108 }
 109
 110
 111 // These are TeX's catcodes
 112 enum CatCode {
 113         catEscape,     // 0    backslash
 114         catBegin,      // 1    {
 115         catEnd,        // 2    }
 116         catMath,       // 3    $
 117         catAlign,      // 4    &
 118         catNewline,    // 5    ^^M
 119         catParameter,  // 6    #
 120         catSuper,      // 7    ^
 121         catSub,        // 8    _
 122         catIgnore,     // 9
 123         catSpace,      // 10   space
 124         catLetter,     // 11   a-zA-Z
 125         catOther,      // 12   none of the above
 126         catActive,     // 13   ~
 127         catComment,    // 14   %
 128         catInvalid     // 15   <delete>
 129 };
 130
 131 CatCode theCatcode[256];
 132
 133
 134 inline CatCode catcode(unsigned char c)
 135 {
 136         return theCatcode[c];
 137 }
 138
 139
 140 enum {
 141         FLAG_BRACE_LAST = 1 << 1,  //  last closing brace ends the parsing process
 142         FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
 143         FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
 144         FLAG_BRACK_END  = 1 << 4,  //  next closing bracket ends the parsing process
 145         FLAG_BOX        = 1 << 5,  //  we are in a box
 146         FLAG_ITEM       = 1 << 6,  //  read a (possibly braced token)
 147         FLAG_BLOCK      = 1 << 7,  //  next block ends the parsing process
 148         FLAG_BLOCK2     = 1 << 8,  //  next block2 ends the parsing process
 149         FLAG_LEAVE      = 1 << 9   //  leave the loop at the end
 150 };
 151
 152
 153 void catInit()
 154 {
 155         fill(theCatcode, theCatcode + 256, catOther);
 156         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
 157         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
 158
 159         theCatcode['\\'] = catEscape;
 160         theCatcode['{']  = catBegin;
 161         theCatcode['}']  = catEnd;
 162         theCatcode['$']  = catMath;
 163         theCatcode['&']  = catAlign;
 164         theCatcode['\n'] = catNewline;
 165         theCatcode['#']  = catParameter;
 166         theCatcode['^']  = catSuper;
 167         theCatcode['_']  = catSub;
 168         theCatcode['\7f'] = catIgnore;
 169         theCatcode[' ']  = catSpace;
 170         theCatcode['\t'] = catSpace;
 171         theCatcode['\r'] = catSpace;
 172         theCatcode['~']  = catActive;
 173         theCatcode['%']  = catComment;
 174 }
 175
 176
 177
 178 //
 179 // Helper class for parsing
 180 //
 181
 182 class Token {
 183 public:
 184         ///
 185         Token() : cs_(), char_(0), cat_(catIgnore) {}
 186         ///
 187         Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
 188         ///
 189         Token(string const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
 190
 191         ///
 192         string const & cs() const { return cs_; }
 193         ///
 194         CatCode cat() const { return cat_; }
 195         ///
 196         char character() const { return char_; }
 197         ///
 198         string asString() const;
 199         ///
 200         bool isCR() const;
 201
 202 private:
 203         ///
 204         string cs_;
 205         ///
 206         char char_;
 207         ///
 208         CatCode cat_;
 209 };
 210
 211 bool Token::isCR() const
 212 {
 213         return cs_ == "\\" || cs_ == "cr" || cs_ == "crcr";
 214 }
 215
 216 string Token::asString() const
 217 {
 218         return cs_.size() ? cs_ : string(1, char_);
 219 }
 220
 221 // Angus' compiler says these are not needed
 222 //bool operator==(Token const & s, Token const & t)
 223 //{
 224 //      return s.character() == t.character()
 225 //              && s.cat() == t.cat() && s.cs() == t.cs();
 226 //}
 227 //
 228 //bool operator!=(Token const & s, Token const & t)
 229 //{
 230 //      return !(s == t);
 231 //}
 232
 233 ostream & operator<<(ostream & os, Token const & t)
 234 {
 235         if (t.cs().size())
 236                 os << "\\" << t.cs();
 237         else
 238                 os << "[" << t.character() << "," << t.cat() << "]";
 239         return os;
 240 }
 241
 242
 243 class Parser {
 244
 245 public:
 246         ///
 247         Parser(LyXLex & lex);
 248         ///
 249         Parser(istream & is);
 250
 251         ///
 252         bool parse_macro(string & name);
 253         ///
 254         bool parse_normal(MathAtom &);
 255         ///
 256         void parse_into(MathArray & array, unsigned flags, MathTextCodes = LM_TC_MIN);
 257         ///
 258         int lineno() const { return lineno_; }
 259         ///
 260         void putback();
 261
 262 private:
 263         ///
 264         void parse_into1(MathArray & array, unsigned flags, MathTextCodes);
 265         ///
 266         string getArg(char lf, char rf);
 267         ///
 268         char getChar();
 269         ///
 270         void error(string const & msg);
 271         ///
 272         bool parse_lines(MathAtom & t, bool numbered, bool outmost);
 273         /// parses {... & ... \\ ... & ... }
 274         bool parse_lines2(MathAtom & t, bool braced);
 275         /// dump contents to screen
 276         void dump() const;
 277
 278 private:
 279         ///
 280         void tokenize(istream & is);
 281         ///
 282         void tokenize(string const & s);
 283         ///
 284         void skipSpaceTokens(istream & is, char c);
 285         ///
 286         void push_back(Token const & t);
 287         ///
 288         void pop_back();
 289         ///
 290         Token const & prevToken() const;
 291         ///
 292         Token const & nextToken() const;
 293         ///
 294         Token const & getToken();
 295         /// skips spaces if any
 296         void skipSpaces();
 297         /// skips opening brace
 298         void skipBegin();
 299         /// skips closing brace
 300         void skipEnd();
 301         /// counts a sequence of hlines
 302         int readHLines();
 303         ///
 304         void lex(string const & s);
 305         ///
 306         bool good() const;
 307
 308         ///
 309         int lineno_;
 310         ///
 311         vector<Token> tokens_;
 312         ///
 313         unsigned pos_;
 314         ///
 315         bool   curr_num_;
 316         ///
 317         string curr_label_;
 318         ///
 319         string curr_skip_;
 320 };
 321
 322
 323 Parser::Parser(LyXLex & lexer)
 324         : lineno_(lexer.getLineNo()), pos_(0), curr_num_(false)
 325 {
 326         tokenize(lexer.getStream());
 327         lexer.eatLine();
 328 }
 329
 330
 331 Parser::Parser(istream & is)
 332         : lineno_(0), pos_(0), curr_num_(false)
 333 {
 334         tokenize(is);
 335 }
 336
 337
 338 void Parser::push_back(Token const & t)
 339 {
 340         tokens_.push_back(t);
 341 }
 342
 343
 344 void Parser::pop_back()
 345 {
 346         tokens_.pop_back();
 347 }
 348
 349
 350 Token const & Parser::prevToken() const
 351 {
 352         static const Token dummy;
 353         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 354 }
 355
 356
 357 Token const & Parser::nextToken() const
 358 {
 359         static const Token dummy;
 360         return good() ? tokens_[pos_] : dummy;
 361 }
 362
 363
 364 Token const & Parser::getToken()
 365 {
 366         static const Token dummy;
 367         //lyxerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
 368         return good() ? tokens_[pos_++] : dummy;
 369 }
 370
 371
 372 void Parser::skipSpaces()
 373 {
 374         while (nextToken().cat() == catSpace)
 375                 getToken();
 376 }
 377
 378
 379 void Parser::skipBegin()
 380 {
 381         if (nextToken().cat() == catBegin)
 382                 getToken();
 383         else
 384                 lyxerr << "'{' expected\n";
 385 }
 386
 387
 388 void Parser::skipEnd()
 389 {
 390         if (nextToken().cat() == catEnd)
 391                 getToken();
 392         else
 393                 lyxerr << "'}' expected\n";
 394 }
 395
 396
 397 int Parser::readHLines()
 398 {
 399         int num = 0;
 400         skipSpaces();
 401         while (nextToken().cs() == "hline") {
 402                 getToken();
 403                 ++num;
 404                 skipSpaces();
 405         }
 406         return num;
 407 }
 408
 409
 410 void Parser::putback()
 411 {
 412         --pos_;
 413 }
 414
 415
 416 bool Parser::good() const
 417 {
 418         return pos_ < tokens_.size();
 419 }
 420
 421
 422 char Parser::getChar()
 423 {
 424         if (!good()) {
 425                 lyxerr << "The input stream is not well..." << endl;
 426                 dump();
 427         }
 428         return tokens_[pos_++].character();
 429 }
 430
 431
 432 string Parser::getArg(char left, char right)
 433 {
 434         skipSpaces();
 435
 436         string result;
 437         char c = getChar();
 438
 439         if (c != left)
 440                 putback();
 441         else
 442                 while ((c = getChar()) != right && good())
 443                         result += c;
 444
 445         return result;
 446 }
 447
 448
 449 void Parser::tokenize(istream & is)
 450 {
 451         // eat everything up to the next \end_inset or end of stream
 452         // and store it in s for further tokenization
 453         string s;
 454         char c;
 455         while (is.get(c)) {
 456                 s += c;
 457                 if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
 458                         s = s.substr(0, s.size() - 10);
 459                         break;
 460                 }
 461         }
 462
 463         // tokenize buffer
 464         tokenize(s);
 465 }
 466
 467
 468 void Parser::skipSpaceTokens(istream & is, char c)
 469 {
 470         // skip trailing spaces
 471         while (catcode(c) == catSpace || catcode(c) == catNewline)
 472                 if (!is.get(c))
 473                         break;
 474         //lyxerr << "putting back: " << c << "\n";
 475         is.putback(c);
 476 }
 477
 478
 479 void Parser::tokenize(string const & buffer)
 480 {
 481         static bool init_done = false;
 482
 483         if (!init_done) {
 484                 catInit();
 485                 init_done = true;
 486         }
 487
 488         istringstream is(buffer.c_str(), ios::in | ios::binary);
 489
 490         char c;
 491         while (is.get(c)) {
 492                 //lyxerr << "reading c: " << c << "\n";
 493
 494                 switch (catcode(c)) {
 495                         case catNewline: {
 496                                 ++lineno_;
 497                                 is.get(c);
 498                                 if (catcode(c) == catNewline)
 499                                         ; //push_back(Token("par"));
 500                                 else {
 501                                         push_back(Token(' ', catSpace));
 502                                         is.putback(c);
 503                                 }
 504                                 break;
 505                         }
 506
 507                         case catComment: {
 508                                 while (is.get(c) && catcode(c) != catNewline)
 509                                         ;
 510                                 ++lineno_;
 511                                 break;
 512                         }
 513
 514                         case catEscape: {
 515                                 is.get(c);
 516                                 if (!is) {
 517                                         error("unexpected end of input");
 518                                 } else {
 519                                         string s(1, c);
 520                                         if (catcode(c) == catLetter) {
 521                                                 // collect letters
 522                                                 while (is.get(c) && catcode(c) == catLetter)
 523                                                         s += c;
 524                                                 skipSpaceTokens(is, c);
 525                                         }
 526                                         push_back(Token(s));
 527                                 }
 528                                 break;
 529                         }
 530
 531                         case catSuper:
 532                         case catSub: {
 533                                 push_back(Token(c, catcode(c)));
 534                                 is.get(c);
 535                                 skipSpaceTokens(is, c);
 536                                 break;
 537                         }
 538
 539                         case catIgnore: {
 540                                 lyxerr << "ignoring a char: " << int(c) << "\n";
 541                                 break;
 542                         }
 543
 544                         default:
 545                                 push_back(Token(c, catcode(c)));
 546                 }
 547         }
 548
 549 #ifdef FILEDEBUG
 550         dump();
 551 #endif
 552 }
 553
 554
 555 void Parser::dump() const
 556 {
 557         lyxerr << "\nTokens: ";
 558         for (unsigned i = 0; i < tokens_.size(); ++i) {
 559                 if (i == pos_)
 560                         lyxerr << " <#> ";
 561                 lyxerr << tokens_[i];
 562         }
 563         lyxerr << "\n";
 564 }
 565
 566
 567 void Parser::error(string const & msg)
 568 {
 569         lyxerr << "Line ~" << lineno_ << ": Math parse error: " << msg << endl;
 570         dump();
 571         //exit(1);
 572 }
 573
 574
 575
 576 bool Parser::parse_lines(MathAtom & t, bool numbered, bool outmost)
 577 {
 578         MathGridInset * p = t->asGridInset();
 579         if (!p) {
 580                 dump();
 581                 lyxerr << "error in Parser::parse_lines() 1\n";
 582                 return false;
 583         }
 584
 585         // save global variables
 586         bool   const saved_num   = curr_num_;
 587         string const saved_label = curr_label_;
 588
 589         // read initial hlines
 590         p->rowinfo(0).lines_ = readHLines();
 591
 592         for (int row = 0; true; ++row) {
 593                 // reset global variables
 594                 curr_num_   = numbered;
 595                 curr_label_.erase();
 596
 597                 // reading a row
 598                 for (MathInset::col_type col = 0; true; ++col) {
 599                         //lyxerr << "reading cell " << row << " " << col << " "
 600                         // << p->ncols() << "\n";
 601                         //lyxerr << "ncols: " << p->ncols() << "\n";
 602
 603                         if (col >= p->ncols()) {
 604                                 //lyxerr << "adding col " << col << "\n";
 605                                 p->addCol(p->ncols());
 606                         }
 607
 608                         MathArray & ar = p->cell(col + row * p->ncols());
 609                         parse_into(ar, FLAG_BLOCK);
 610                         // remove 'unnecessary' braces:
 611                         if (ar.size() == 1 && ar.back()->asBraceInset())
 612                                 ar = ar.back()->asBraceInset()->cell(0);
 613                         //lyxerr << "ar: " << ar << "\n";
 614
 615                         // break if cell is not followed by an ampersand
 616                         if (nextToken().cat() != catAlign) {
 617                                 //lyxerr << "less cells read than normal in row/col: "
 618                                 //      << row << " " << col << "\n";
 619                                 break;
 620                         }
 621
 622                         // skip the ampersand
 623                         getToken();
 624                 }
 625
 626                 if (outmost) {
 627                         MathHullInset * m = t->asHullInset();
 628                         if (!m) {
 629                                 lyxerr << "error in Parser::parse_lines() 2\n";
 630                                 return false;
 631                         }
 632                         m->numbered(row, curr_num_);
 633                         m->label(row, curr_label_);
 634                         if (curr_skip_.size()) {
 635                                 m->vcrskip(LyXLength(curr_skip_), row);
 636                                 curr_skip_.erase();
 637                         }
 638                 }
 639
 640                 // is a \\ coming?
 641                 if (nextToken().isCR()) {
 642                         // skip the cr-token
 643                         getToken();
 644
 645                         // try to read a length
 646                         //get
 647
 648                         // read hlines for next row
 649                         p->rowinfo(row + 1).lines_ = readHLines();
 650                 }
 651
 652                 // we are finished if the next token is an 'end'
 653                 if (nextToken().cs() == "end") {
 654                         // skip the end-token
 655                         getToken();
 656                         getArg('{','}');
 657
 658                         // leave the 'read a line'-loop
 659                         break;
 660                 }
 661
 662                 // otherwise, we have to start a new row
 663                 p->appendRow();
 664         }
 665
 666         // restore "global" variables
 667         curr_num_   = saved_num;
 668         curr_label_ = saved_label;
 669
 670         return true;
 671 }
 672
 673
 674 bool Parser::parse_lines2(MathAtom & t, bool braced)
 675 {
 676         MathGridInset * p = t->asGridInset();
 677         if (!p) {
 678                 lyxerr << "error in Parser::parse_lines() 1\n";
 679                 return false;
 680         }
 681
 682         for (int row = 0; true; ++row) {
 683                 // reading a row
 684                 for (MathInset::col_type col = 0; true; ++col) {
 685                         //lyxerr << "reading cell " << row << " " << col << " " << p->ncols() << "\n";
 686
 687                         if (col >= p->ncols()) {
 688                                 //lyxerr << "adding col " << col << "\n";
 689                                 p->addCol(p->ncols());
 690                         }
 691
 692                         parse_into(p->cell(col + row * p->ncols()), FLAG_BLOCK2);
 693                         //lyxerr << "read cell: " << p->cell(col + row * p->ncols()) << "\n";
 694
 695                         // break if cell is not followed by an ampersand
 696                         if (nextToken().cat() != catAlign) {
 697                                 //lyxerr << "less cells read than normal in row/col: " << row << " " << col << "\n";
 698                                 break;
 699                         }
 700
 701                         // skip the ampersand
 702                         getToken();
 703                 }
 704
 705                 // is a \\ coming?
 706                 if (nextToken().isCR()) {
 707                         // skip the cr-token
 708                         getToken();
 709                 }
 710
 711                 // we are finished if the next token is the one we expected
 712                 // skip the end-token
 713                 // leave the 'read a line'-loop
 714                 if (braced) {
 715                         if (nextToken().cat() == catEnd) {
 716                                 getToken();
 717                                 break;
 718                         }
 719                 } else {
 720                         if (nextToken().cs() == "end") {
 721                                 getToken();
 722                                 getArg('{','}');
 723                                 break;
 724                         }
 725                 }
 726
 727                 // otherwise, we have to start a new row
 728                 p->appendRow();
 729         }
 730
 731         return true;
 732 }
 733
 734
 735
 736
 737 bool Parser::parse_macro(string & name)
 738 {
 739         int nargs = 0;
 740         name = "{error}";
 741         skipSpaces();
 742
 743         if (nextToken().cs() == "def") {
 744
 745                 getToken();
 746                 name = getToken().cs();
 747
 748                 string pars;
 749                 while (good() && nextToken().cat() != catBegin)
 750                         pars += getToken().cs();
 751
 752                 if (!good()) {
 753                         lyxerr << "bad stream in parse_macro\n";
 754                         dump();
 755                         return false;
 756                 }
 757
 758                 //lyxerr << "read \\def parameter list '" << pars << "'\n";
 759                 if (!pars.empty()) {
 760                         lyxerr << "can't handle non-empty parameter lists\n";
 761                         dump();
 762                         return false;
 763                 }
 764
 765         } else if (nextToken().cs() == "newcommand") {
 766
 767                 getToken();
 768
 769                 if (getToken().cat() != catBegin) {
 770                         lyxerr << "'{' in \\newcommand expected (1) \n";
 771                         dump();
 772                         return false;
 773                 }
 774
 775                 name = getToken().cs();
 776
 777                 if (getToken().cat() != catEnd) {
 778                         lyxerr << "'}' expected\n";
 779                         return false;
 780                 }
 781
 782                 string arg  = getArg('[', ']');
 783                 if (!arg.empty())
 784                         nargs = atoi(arg.c_str());
 785
 786         } else {
 787                 lyxerr << "\\newcommand or \\def  expected\n";
 788                 return false;
 789         }
 790
 791
 792         if (getToken().cat() != catBegin) {
 793                 lyxerr << "'{' in macro definition expected (2)\n";
 794                 return false;
 795         }
 796
 797         MathArray ar1;
 798         parse_into(ar1, FLAG_BRACE_LAST);
 799
 800         // we cannot handle recursive stuff at all
 801         MathArray test;
 802         test.push_back(createMathInset(name));
 803         if (ar1.contains(test)) {
 804                 lyxerr << "we cannot handle recursive macros at all.\n";
 805                 return false;
 806         }
 807
 808         // is a version for display attached?
 809         MathArray ar2;
 810         parse_into(ar2, FLAG_ITEM);
 811
 812         MathMacroTable::create(name, nargs, ar1, ar2);
 813         return true;
 814 }
 815
 816
 817 bool Parser::parse_normal(MathAtom & matrix)
 818 {
 819         skipSpaces();
 820         Token const & t = getToken();
 821
 822         if (t.cs() == "(") {
 823                 matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 824                 parse_into(matrix->cell(0), 0);
 825                 return true;
 826         }
 827
 828         if (t.cat() == catMath) {
 829                 Token const & n = getToken();
 830                 if (n.cat() == catMath) {
 831                         // TeX's $$...$$ syntax for displayed math
 832                         matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 833                         MathHullInset * p = matrix->asHullInset();
 834                         parse_into(p->cell(0), 0);
 835                         p->numbered(0, curr_num_);
 836                         p->label(0, curr_label_);
 837                 } else {
 838                         // simple $...$  stuff
 839                         putback();
 840                         matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 841                         parse_into(matrix->cell(0), 0);
 842                 }
 843                 return true;
 844         }
 845
 846         if (!t.cs().size()) {
 847                 lyxerr << "start of math expected, got '" << t << "'\n";
 848                 return false;
 849         }
 850
 851         string const & cs = t.cs();
 852
 853         if (cs == "[") {
 854                 curr_num_ = 0;
 855                 curr_label_.erase();
 856                 matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 857                 MathHullInset * p = matrix->asHullInset();
 858                 parse_into(p->cell(0), 0);
 859                 p->numbered(0, curr_num_);
 860                 p->label(0, curr_label_);
 861                 return true;
 862         }
 863
 864         if (cs != "begin") {
 865                 lyxerr[Debug::MATHED]
 866                         << "'begin' of un-simple math expected, got '" << cs << "'\n";
 867                 return false;
 868         }
 869
 870         string const name = getArg('{', '}');
 871
 872         if (name == "math") {
 873                 matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 874                 parse_into(matrix->cell(0), 0);
 875                 return true;
 876         }
 877
 878         if (name == "equation" || name == "equation*" || name == "displaymath") {
 879                 curr_num_ = (name == "equation");
 880                 curr_label_.erase();
 881                 matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 882                 MathHullInset * p = matrix->asHullInset();
 883                 parse_into(p->cell(0), FLAG_END);
 884                 p->numbered(0, curr_num_);
 885                 p->label(0, curr_label_);
 886                 return true;
 887         }
 888
 889         if (name == "eqnarray" || name == "eqnarray*") {
 890                 matrix = MathAtom(new MathHullInset(LM_OT_EQNARRAY));
 891                 return parse_lines(matrix, !stared(name), true);
 892         }
 893
 894         if (name == "align" || name == "align*") {
 895                 matrix = MathAtom(new MathHullInset(LM_OT_ALIGN));
 896                 return parse_lines(matrix, !stared(name), true);
 897         }
 898
 899         if (name == "alignat" || name == "alignat*") {
 900                 int nc = 2 * atoi(getArg('{', '}').c_str());
 901                 matrix = MathAtom(new MathHullInset(LM_OT_ALIGNAT, nc));
 902                 return parse_lines(matrix, !stared(name), true);
 903         }
 904
 905         if (name == "xalignat" || name == "xalignat*") {
 906                 int nc = 2 * atoi(getArg('{', '}').c_str());
 907                 matrix = MathAtom(new MathHullInset(LM_OT_XALIGNAT, nc));
 908                 return parse_lines(matrix, !stared(name), true);
 909         }
 910
 911         if (name == "xxalignat") {
 912                 int nc = 2 * atoi(getArg('{', '}').c_str());
 913                 matrix = MathAtom(new MathHullInset(LM_OT_XXALIGNAT, nc));
 914                 return parse_lines(matrix, !stared(name), true);
 915         }
 916
 917         if (name == "multline" || name == "multline*") {
 918                 matrix = MathAtom(new MathHullInset(LM_OT_MULTLINE));
 919                 return parse_lines(matrix, !stared(name), true);
 920         }
 921
 922         if (name == "gather" || name == "gather*") {
 923                 matrix = MathAtom(new MathHullInset(LM_OT_GATHER));
 924                 return parse_lines(matrix, !stared(name), true);
 925         }
 926
 927         lyxerr[Debug::MATHED] << "1: unknown math environment: " << name << "\n";
 928         lyxerr << "1: unknown math environment: " << name << "\n";
 929         return false;
 930 }
 931
 932
 933 void Parser::parse_into(MathArray & array, unsigned flags, MathTextCodes code)
 934 {
 935         parse_into1(array, flags, code);
 936         // remove 'unnecessary' braces:
 937         if (array.size() == 1 && array.back()->asBraceInset()) {
 938                 lyxerr << "extra braces removed\n";
 939                 array = array.back()->asBraceInset()->cell(0);
 940         }
 941 }
 942
 943
 944 void Parser::parse_into1(MathArray & array, unsigned flags, MathTextCodes code)
 945 {
 946         bool panic  = false;
 947         int  limits = 0;
 948
 949         while (good()) {
 950                 Token const & t = getToken();
 951
 952 #ifdef FILEDEBUG
 953                 lyxerr << "t: " << t << " flags: " << flags << "\n";
 954                 //array.dump();
 955                 lyxerr << "\n";
 956 #endif
 957
 958                 if (flags & FLAG_ITEM) {
 959                         if (t.cat() == catSpace)
 960                                 continue;
 961
 962                         flags &= ~FLAG_ITEM;
 963                         if (t.cat() == catBegin) {
 964                                 // skip the brace and collect everything to the next matching
 965                                 // closing brace
 966                                 flags |= FLAG_BRACE_LAST;
 967                                 continue;
 968                         }
 969
 970                         // handle only this single token, leave the loop if done
 971                         flags |= FLAG_LEAVE;
 972                 }
 973
 974                 if (flags & FLAG_BLOCK) {
 975                         if (t.cat() == catAlign || t.isCR() || t.cs() == "end") {
 976                                 putback();
 977                                 return;
 978                         }
 979                 }
 980
 981                 if (flags & FLAG_BLOCK2) {
 982                         if (t.cat() == catAlign || t.isCR() || t.cs() == "end"
 983                                         || t.cat() == catEnd) {
 984                                 putback();
 985                                 return;
 986                         }
 987                 }
 988
 989                 //
 990                 // cat codes
 991                 //
 992                 if (t.cat() == catMath) {
 993                         if (flags & FLAG_BOX) {
 994                                 // we are inside an mbox, so opening new math is allowed
 995                                 array.push_back(MathAtom(new MathHullInset(LM_OT_SIMPLE)));
 996                                 parse_into(array.back()->cell(0), 0);
 997                         } else {
 998                                 // otherwise this is the end of the formula
 999                                 break;
1000                         }
1001                 }
1002
1003                 else if (t.cat() == catLetter)
1004                         add(array, t.character(), code);
1005
1006                 else if (t.cat() == catSpace && code == LM_TC_TEXTRM)
1007                         add(array, t.character(), code);
1008
1009                 else if (t.cat() == catParameter) {
1010                         Token const & n = getToken();
1011                         array.push_back(MathAtom(new MathMacroArgument(n.character()-'0', code)));
1012                 }
1013
1014                 else if (t.cat() == catBegin) {
1015                         MathArray ar;
1016                         parse_into(ar, FLAG_BRACE_LAST);
1017 #ifndef WITH_WARNINGS
1018 #warning this might be wrong in general!
1019 #endif
1020                         // ignore braces around simple items
1021                         if ((ar.size() == 1 && !ar.front()->needsBraces()
1022        || (ar.size() == 2 && !ar.front()->needsBraces()
1023                                             && ar.back()->asScriptInset()))
1024        || (ar.size() == 0 && array.size() == 0))
1025                         {
1026                                 array.push_back(ar);
1027                         } else {
1028                                 array.push_back(MathAtom(new MathBraceInset));
1029                                 array.back()->cell(0).swap(ar);
1030                         }
1031                 }
1032
1033                 else if (t.cat() == catEnd) {
1034                         if (flags & FLAG_BRACE_LAST)
1035                                 return;
1036                         lyxerr << "found '}' unexpectedly, array: '" << array << "'\n";
1037                         dump();
1038                         //lyxerr << "found '}' unexpectedly\n";
1039                         //lyx::Assert(0);
1040                         //add(array, '}', LM_TC_TEX);
1041                 }
1042
1043                 else if (t.cat() == catAlign) {
1044                         lyxerr << "found tab unexpectedly, array: '" << array << "'\n";
1045                         dump();
1046                         //lyxerr << "found tab unexpectedly\n";
1047                         add(array, '&', LM_TC_TEX);
1048                 }
1049
1050                 else if (t.cat() == catSuper || t.cat() == catSub) {
1051                         bool up = (t.cat() == catSuper);
1052                         MathScriptInset * p = 0;
1053                         if (array.size())
1054                                 p = array.back()->asScriptInset();
1055                         if (!p || p->has(up)) {
1056                                 array.push_back(MathAtom(new MathScriptInset(up)));
1057                                 p = array.back()->asScriptInset();
1058                         }
1059                         p->ensure(up);
1060                         parse_into(p->cell(up), FLAG_ITEM);
1061                         p->limits(limits);
1062                         limits = 0;
1063                 }
1064
1065                 else if (t.character() == ']' && (flags & FLAG_BRACK_END))
1066                         return;
1067
1068                 else if (t.cat() == catOther)
1069                         add(array, t.character(), code);
1070
1071                 //
1072                 // control sequences
1073                 //
1074                 else if (t.cs() == "protect")
1075                         // ignore \\protect, will be re-added during output
1076                         ;
1077
1078                 else if (t.cs() == "end")
1079                         break;
1080
1081                 else if (t.cs() == ")")
1082                         break;
1083
1084                 else if (t.cs() == "]")
1085                         break;
1086
1087                 else if (t.cs() == "\\") {
1088                         curr_skip_ = getArg('[', ']');
1089                         //lyxerr << "found newline unexpectedly, array: '" << array << "'\n";
1090                         lyxerr << "found newline unexpectedly\n";
1091                         array.push_back(createMathInset("\\"));
1092                 }
1093
1094                 else if (t.cs() == "limits")
1095                         limits = 1;
1096
1097                 else if (t.cs() == "nolimits")
1098                         limits = -1;
1099
1100                 else if (t.cs() == "nonumber")
1101                         curr_num_ = false;
1102
1103                 else if (t.cs() == "number")
1104                         curr_num_ = true;
1105
1106                 else if (t.cs() == "sqrt") {
1107                         char c = getChar();
1108                         if (c == '[') {
1109                                 array.push_back(MathAtom(new MathRootInset));
1110                                 parse_into(array.back()->cell(0), FLAG_BRACK_END);
1111                                 parse_into(array.back()->cell(1), FLAG_ITEM);
1112                         } else {
1113                                 putback();
1114                                 array.push_back(MathAtom(new MathSqrtInset));
1115                                 parse_into(array.back()->cell(0), FLAG_ITEM);
1116                         }
1117                 }
1118
1119                 else if (t.cs() == "left") {
1120                         string l = getToken().asString();
1121                         MathArray ar;
1122                         parse_into(ar, FLAG_RIGHT);
1123                         string r = getToken().asString();
1124                         MathAtom dl(new MathDelimInset(l, r));
1125                         dl->cell(0) = ar;
1126                         array.push_back(dl);
1127                 }
1128
1129                 else if (t.cs() == "right") {
1130                         if (!(flags & FLAG_RIGHT)) {
1131                                 //lyxerr << "got so far: '" << array << "'\n";
1132                                 error("Unmatched right delimiter");
1133                         }
1134                         return;
1135                 }
1136
1137                 else if (t.cs() == "begin") {
1138                         string const name = getArg('{', '}');
1139                         if (name == "array" || name == "subarray") {
1140                                 string const valign = getArg('[', ']') + 'c';
1141                                 string const halign = getArg('{', '}');
1142                                 array.push_back(MathAtom(new MathArrayInset(name, valign[0], halign)));
1143                                 parse_lines(array.back(), false, false);
1144                         } else if (name == "split" || name == "cases" ||
1145                                          name == "gathered" || name == "aligned") {
1146                                 array.push_back(createMathInset(name));
1147                                 parse_lines(array.back(), false, false);
1148                         } else if (name == "matrix"  || name == "pmatrix" || name == "bmatrix" ||
1149                                          name == "vmatrix" || name == "Vmatrix") {
1150                                 array.push_back(createMathInset(name));
1151                                 parse_lines2(array.back(), false);
1152                         } else
1153                                 lyxerr << "unknow math inset begin '" << name << "'\n";
1154                 }
1155
1156                 else if (t.cs() == "kern") {
1157 #ifdef WITH_WARNINGS
1158 #warning A hack...
1159 #endif
1160                         string s;
1161                         while (1) {
1162                                 Token const & t = getToken();
1163                                 if (!good()) {
1164                                         putback();
1165                                         break;
1166                                 }
1167                                 s += t.character();
1168                                 if (isValidLength(s))
1169                                         break;
1170                         }
1171                         array.push_back(MathAtom(new MathKernInset(s)));
1172                 }
1173
1174 /*
1175                 else if (t.cs() == "lyxkern") {
1176                         MathAtom p = createMathInset(t.cs());
1177                         parse_into(p->cell(0), flags, code);
1178                         array.push_back(p);
1179                 }
1180 */
1181
1182                 else if (t.cs() == "label") {
1183                         curr_label_ = getArg('{', '}');
1184                 }
1185
1186                 else if (t.cs() == "choose" || t.cs() == "over" || t.cs() == "atop") {
1187                         MathAtom p = createMathInset(t.cs());
1188                         array.swap(p->cell(0));
1189                         parse_into(p->cell(1), flags, code);
1190                         array.push_back(p);
1191                         return;
1192                 }
1193
1194                 else if (t.cs() == "substack") {
1195                         array.push_back(createMathInset(t.cs()));
1196                         skipBegin();
1197                         parse_lines2(array.back(), true);
1198                 }
1199
1200                 else if (t.cs() == "xymatrix") {
1201                         array.push_back(createMathInset(t.cs()));
1202                         skipBegin();
1203                         parse_lines2(array.back(), true);
1204                 }
1205
1206 #if 0
1207                 // Disabled
1208                 else if (1 && t.cs() == "ar") {
1209                         MathXYArrowInset * p = new MathXYArrowInset;
1210
1211                         // try to read target
1212                         char c = getChar();
1213                         if (c == '[') {
1214                                 parse_into(p->cell(0), FLAG_BRACK_END);
1215                                 //lyxerr << "read target: " << p->cell(0) << "\n";
1216                         } else {
1217                                 putback();
1218                         }
1219
1220                         // try to read label
1221                         if (nextToken().cat() == catSuper || nextToken().cat() == catSub) {
1222                                 p->up_ = nextToken().cat() == catSuper;
1223                                 getToken();
1224                                 parse_into(p->cell(1), FLAG_ITEM);
1225                                 //lyxerr << "read label: " << p->cell(1) << "\n";
1226                         }
1227
1228                         array.push_back(MathAtom(p));
1229                         //lyxerr << "read array: " << array << "\n";
1230                 }
1231 #endif
1232
1233 #if 0
1234                 else if (t.cs() == "mbox" || t.cs() == "text") {
1235                         //array.push_back(createMathInset(t.cs()));
1236                         array.push_back(MathAtom(new MathBoxInset(t.cs())));
1237                         // slurp in the argument of mbox
1238
1239                         MathBoxInset * p = array.back()->asBoxInset();
1240                         //lyx::assert(p);
1241                 }
1242 #endif
1243
1244
1245                 else if (t.cs().size()) {
1246                         latexkeys const * l = in_word_set(t.cs());
1247                         if (l) {
1248                                 if (l->token == LM_TK_FONT) {
1249                                         //lyxerr << "starting font\n";
1250                                         //CatCode catSpaceSave = theCatcode[' '];
1251                                         //if (l->id == LM_TC_TEXTRM) {
1252                                         //      // temporarily change catcode
1253                                         //      theCatcode[' '] = catLetter;
1254                                         //}
1255
1256                                         MathArray ar;
1257                                         parse_into(ar, FLAG_ITEM, static_cast<MathTextCodes>(l->id));
1258                                         array.push_back(ar);
1259
1260                                         // undo catcode changes
1261                                         ////theCatcode[' '] = catSpaceSave;
1262                                         //lyxerr << "ending font\n";
1263                                 }
1264
1265                                 else if (l->token == LM_TK_OLDFONT) {
1266                                         code = static_cast<MathTextCodes>(l->id);
1267                                 }
1268
1269                                 else if (l->token == LM_TK_BOX) {
1270                                         MathAtom p = createMathInset(t.cs());
1271                                         parse_into(p->cell(0), FLAG_ITEM | FLAG_BOX, LM_TC_BOX);
1272                                         array.push_back(p);
1273                                 }
1274
1275                                 else if (l->token == LM_TK_STY) {
1276                                         MathAtom p = createMathInset(t.cs());
1277                                         parse_into(p->cell(0), flags, code);
1278                                         array.push_back(p);
1279                                         return;
1280                                 }
1281
1282                                 else {
1283                                         MathAtom p = createMathInset(t.cs());
1284                                         for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
1285                                                 parse_into(p->cell(i), FLAG_ITEM);
1286                                         array.push_back(p);
1287                                 }
1288                         }
1289
1290                         else {
1291                                 MathAtom p = createMathInset(t.cs());
1292                                 for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
1293                                         parse_into(p->cell(i), FLAG_ITEM);
1294                                 array.push_back(p);
1295                         }
1296                 }
1297
1298
1299                 if (flags & FLAG_LEAVE) {
1300                         flags &= ~FLAG_LEAVE;
1301                         break;
1302                 }
1303         }
1304
1305         if (panic) {
1306                 lyxerr << " Math Panic, expect problems!\n";
1307                 //   Search for the end command.
1308                 Token t;
1309                 do {
1310                         t = getToken();
1311                 } while (good() && t.cs() != "end");
1312         }
1313 }
1314
1315
1316
1317 } // anonymous namespace
1318
1319
1320 void mathed_parse_cell(MathArray & ar, string const & str)
1321 {
1322         istringstream is(str.c_str());
1323         mathed_parse_cell(ar, is);
1324 }
1325
1326
1327 void mathed_parse_cell(MathArray & ar, istream & is)
1328 {
1329         Parser(is).parse_into(ar, 0);
1330 }
1331
1332
1333
1334 bool mathed_parse_macro(string & name, string const & str)
1335 {
1336         istringstream is(str.c_str());
1337         Parser parser(is);
1338         return parser.parse_macro(name);
1339 }
1340
1341 bool mathed_parse_macro(string & name, istream & is)
1342 {
1343         Parser parser(is);
1344         return parser.parse_macro(name);
1345 }
1346
1347 bool mathed_parse_macro(string & name, LyXLex & lex)
1348 {
1349         Parser parser(lex);
1350         return parser.parse_macro(name);
1351 }
1352
1353
1354
1355 bool mathed_parse_normal(MathAtom & t, string const & str)
1356 {
1357         istringstream is(str.c_str());
1358         Parser parser(is);
1359         return parser.parse_normal(t);
1360 }
1361
1362 bool mathed_parse_normal(MathAtom & t, istream & is)
1363 {
1364         Parser parser(is);
1365         return parser.parse_normal(t);
1366 }
1367
1368 bool mathed_parse_normal(MathAtom & t, LyXLex & lex)
1369 {
1370         Parser parser(lex);
1371         return parser.parse_normal(t);
1372 }