src/mathed/math_parser.C

   1 /*
   2  *  File:        math_parser.C
   3  *  Purpose:     Parser for mathed
   4  *  Author:      Alejandro Aguilar Sierra <asierra@servidor.unam.mx>
   5  *  Created:     January 1996
   6  *  Description: Parse LaTeX2e math mode code.
   7  *
   8  *  Dependencies: Xlib, XForms
   9  *
  10  *  Copyright: 1996, Alejandro Aguilar Sierra
  11  *
  12  *   Version: 0.8beta.
  13  *
  14  *   You are free to use and modify this code under the terms of
  15  *   the GNU General Public Licence version 2 or later.
  16  */
  17
  18 /*
  19
  20 If someone desperately needs partial "structures" (such as a few cells of
  21 an array inset or similar) (s)he could uses the following hack as starting
  22 point to write some macros:
  23
  24   \newif\ifcomment
  25   \commentfalse
  26   \ifcomment
  27           \def\makeamptab{\catcode`\&=4\relax}
  28           \def\makeampletter{\catcode`\&=11\relax}
  29     \def\b{\makeampletter\expandafter\makeamptab\bi}
  30     \long\def\bi#1\e{}
  31   \else
  32     \def\b{}\def\e{}
  33   \fi
  34
  35   ...
  36
  37   \[\begin{array}{ccc}
  38    1 & 2\b & 3^2\\
  39    4 & 5\e & 6\\
  40    7 & 8 & 9
  41   \end{array}\]
  42
  43 */
  44
  45
  46 #include <config.h>
  47
  48 #ifdef __GNUG__
  49 #pragma implementation
  50 #endif
  51
  52 #include "math_parser.h"
  53 #include "math_inset.h"
  54 #include "math_arrayinset.h"
  55 #include "math_braceinset.h"
  56 #include "math_boxinset.h"
  57 #include "math_charinset.h"
  58 #include "math_deliminset.h"
  59 #include "math_factory.h"
  60 #include "math_funcinset.h"
  61 #include "math_kerninset.h"
  62 #include "math_macro.h"
  63 #include "math_macrotable.h"
  64 #include "math_macrotemplate.h"
  65 #include "math_hullinset.h"
  66 #include "math_rootinset.h"
  67 #include "math_sizeinset.h"
  68 #include "math_sqrtinset.h"
  69 #include "math_scriptinset.h"
  70 #include "math_specialcharinset.h"
  71 #include "math_sqrtinset.h"
  72 #include "math_support.h"
  73 #include "math_xyarrowinset.h"
  74
  75 #include "lyxlex.h"
  76 #include "debug.h"
  77 #include "support/LAssert.h"
  78 #include "support/lstrings.h"
  79
  80 #include <cctype>
  81 #include <stack>
  82 #include <algorithm>
  83
  84 using std::istream;
  85 using std::ostream;
  86 using std::ios;
  87 using std::endl;
  88 using std::stack;
  89 using std::fill;
  90 using std::vector;
  91
  92 //#define FILEDEBUG
  93
  94
  95 namespace {
  96
  97 bool stared(string const & s)
  98 {
  99         string::size_type const n = s.size();
 100         return n && s[n - 1] == '*';
 101 }
 102
 103
 104 void add(MathArray & ar, char c, MathTextCodes code)
 105 {
 106         ar.push_back(MathAtom(new MathCharInset(c, code)));
 107 }
 108
 109
 110 // These are TeX's catcodes
 111 enum CatCode {
 112         catEscape,     // 0    backslash
 113         catBegin,      // 1    {
 114         catEnd,        // 2    }
 115         catMath,       // 3    $
 116         catAlign,      // 4    &
 117         catNewline,    // 5    ^^M
 118         catParameter,  // 6    #
 119         catSuper,      // 7    ^
 120         catSub,        // 8    _
 121         catIgnore,     // 9
 122         catSpace,      // 10   space
 123         catLetter,     // 11   a-zA-Z
 124         catOther,      // 12   none of the above
 125         catActive,     // 13   ~
 126         catComment,    // 14   %
 127         catInvalid     // 15   <delete>
 128 };
 129
 130 CatCode theCatcode[256];
 131
 132
 133 inline CatCode catcode(unsigned char c)
 134 {
 135         return theCatcode[c];
 136 }
 137
 138
 139 enum {
 140         FLAG_BRACE_LAST = 1 << 1,  //  last closing brace ends the parsing process
 141         FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
 142         FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
 143         FLAG_BRACK_END  = 1 << 4,  //  next closing bracket ends the parsing process
 144         FLAG_BOX        = 1 << 5,  //  we are in a box
 145         FLAG_ITEM       = 1 << 6,  //  read a (possibly braced token)
 146         FLAG_BLOCK      = 1 << 7,  //  next block ends the parsing process
 147         FLAG_BLOCK2     = 1 << 8,  //  next block2 ends the parsing process
 148         FLAG_LEAVE      = 1 << 9   //  leave the loop at the end
 149 };
 150
 151
 152 void catInit()
 153 {
 154         fill(theCatcode, theCatcode + 256, catOther);
 155         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
 156         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
 157
 158         theCatcode['\\'] = catEscape;
 159         theCatcode['{']  = catBegin;
 160         theCatcode['}']  = catEnd;
 161         theCatcode['$']  = catMath;
 162         theCatcode['&']  = catAlign;
 163         theCatcode['\n'] = catNewline;
 164         theCatcode['#']  = catParameter;
 165         theCatcode['^']  = catSuper;
 166         theCatcode['_']  = catSub;
 167         theCatcode['\7f'] = catIgnore;
 168         theCatcode[' ']  = catSpace;
 169         theCatcode['\t'] = catSpace;
 170         theCatcode['\r'] = catSpace;
 171         theCatcode['~']  = catActive;
 172         theCatcode['%']  = catComment;
 173 }
 174
 175
 176
 177 //
 178 // Helper class for parsing
 179 //
 180
 181 class Token {
 182 public:
 183         ///
 184         Token() : cs_(), char_(0), cat_(catIgnore) {}
 185         ///
 186         Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
 187         ///
 188         Token(string const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
 189
 190         ///
 191         string const & cs() const { return cs_; }
 192         ///
 193         CatCode cat() const { return cat_; }
 194         ///
 195         char character() const { return char_; }
 196         ///
 197         string asString() const;
 198         ///
 199         bool isCR() const;
 200
 201 private:
 202         ///
 203         string cs_;
 204         ///
 205         char char_;
 206         ///
 207         CatCode cat_;
 208 };
 209
 210 bool Token::isCR() const
 211 {
 212         return cs_ == "\\" || cs_ == "cr" || cs_ == "crcr";
 213 }
 214
 215 string Token::asString() const
 216 {
 217         return cs_.size() ? cs_ : string(1, char_);
 218 }
 219
 220 // Angus' compiler says these are not needed
 221 //bool operator==(Token const & s, Token const & t)
 222 //{
 223 //      return s.character() == t.character()
 224 //              && s.cat() == t.cat() && s.cs() == t.cs();
 225 //}
 226 //
 227 //bool operator!=(Token const & s, Token const & t)
 228 //{
 229 //      return !(s == t);
 230 //}
 231
 232 ostream & operator<<(ostream & os, Token const & t)
 233 {
 234         if (t.cs().size())
 235                 os << "\\" << t.cs();
 236         else
 237                 os << "[" << t.character() << "," << t.cat() << "]";
 238         return os;
 239 }
 240
 241
 242 class Parser {
 243
 244 public:
 245         ///
 246         Parser(LyXLex & lex);
 247         ///
 248         Parser(istream & is);
 249
 250         ///
 251         bool parse_macro(string & name);
 252         ///
 253         bool parse_normal(MathAtom &);
 254         ///
 255         void parse_into(MathArray & array, unsigned flags, MathTextCodes = LM_TC_MIN);
 256         ///
 257         int lineno() const { return lineno_; }
 258         ///
 259         void putback();
 260
 261 private:
 262         ///
 263         void parse_into1(MathArray & array, unsigned flags, MathTextCodes);
 264         ///
 265         string getArg(char lf, char rf);
 266         ///
 267         char getChar();
 268         ///
 269         void error(string const & msg);
 270         ///
 271         bool parse_lines(MathAtom & t, bool numbered, bool outmost);
 272         /// parses {... & ... \\ ... & ... }
 273         bool parse_lines2(MathAtom & t, bool braced);
 274         /// dump contents to screen
 275         void dump() const;
 276
 277 private:
 278         ///
 279         void tokenize(istream & is);
 280         ///
 281         void tokenize(string const & s);
 282         ///
 283         void skipSpaceTokens(istream & is, char c);
 284         ///
 285         void push_back(Token const & t);
 286         ///
 287         void pop_back();
 288         ///
 289         Token const & prevToken() const;
 290         ///
 291         Token const & nextToken() const;
 292         ///
 293         Token const & getToken();
 294         /// skips spaces if any
 295         void skipSpaces();
 296         /// skips opening brace
 297         void skipBegin();
 298         /// skips closing brace
 299         void skipEnd();
 300         /// counts a sequence of hlines
 301         int readHLines();
 302         ///
 303         void lex(string const & s);
 304         ///
 305         bool good() const;
 306
 307         ///
 308         int lineno_;
 309         ///
 310         vector<Token> tokens_;
 311         ///
 312         unsigned pos_;
 313         ///
 314         bool   curr_num_;
 315         ///
 316         string curr_label_;
 317         ///
 318         string curr_skip_;
 319 };
 320
 321
 322 Parser::Parser(LyXLex & lexer)
 323         : lineno_(lexer.getLineNo()), pos_(0), curr_num_(false)
 324 {
 325         tokenize(lexer.getStream());
 326         lexer.eatLine();
 327 }
 328
 329
 330 Parser::Parser(istream & is)
 331         : lineno_(0), pos_(0), curr_num_(false)
 332 {
 333         tokenize(is);
 334 }
 335
 336
 337 void Parser::push_back(Token const & t)
 338 {
 339         tokens_.push_back(t);
 340 }
 341
 342
 343 void Parser::pop_back()
 344 {
 345         tokens_.pop_back();
 346 }
 347
 348
 349 Token const & Parser::prevToken() const
 350 {
 351         static const Token dummy;
 352         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 353 }
 354
 355
 356 Token const & Parser::nextToken() const
 357 {
 358         static const Token dummy;
 359         return good() ? tokens_[pos_] : dummy;
 360 }
 361
 362
 363 Token const & Parser::getToken()
 364 {
 365         static const Token dummy;
 366         //lyxerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
 367         return good() ? tokens_[pos_++] : dummy;
 368 }
 369
 370
 371 void Parser::skipSpaces()
 372 {
 373         while (nextToken().cat() == catSpace)
 374                 getToken();
 375 }
 376
 377
 378 void Parser::skipBegin()
 379 {
 380         if (nextToken().cat() == catBegin)
 381                 getToken();
 382         else
 383                 lyxerr << "'{' expected\n";
 384 }
 385
 386
 387 void Parser::skipEnd()
 388 {
 389         if (nextToken().cat() == catEnd)
 390                 getToken();
 391         else
 392                 lyxerr << "'}' expected\n";
 393 }
 394
 395
 396 int Parser::readHLines()
 397 {
 398         int num = 0;
 399         skipSpaces();
 400         while (nextToken().cs() == "hline") {
 401                 getToken();
 402                 ++num;
 403                 skipSpaces();
 404         }
 405         return num;
 406 }
 407
 408
 409 void Parser::putback()
 410 {
 411         --pos_;
 412 }
 413
 414
 415 bool Parser::good() const
 416 {
 417         return pos_ < tokens_.size();
 418 }
 419
 420
 421 char Parser::getChar()
 422 {
 423         if (!good())
 424                 lyxerr << "The input stream is not well..." << endl;
 425         return tokens_[pos_++].character();
 426 }
 427
 428
 429 string Parser::getArg(char left, char right)
 430 {
 431         skipSpaces();
 432
 433         string result;
 434         char c = getChar();
 435
 436         if (c != left)
 437                 putback();
 438         else
 439                 while ((c = getChar()) != right && good())
 440                         result += c;
 441
 442         return result;
 443 }
 444
 445
 446 void Parser::tokenize(istream & is)
 447 {
 448         // eat everything up to the next \end_inset or end of stream
 449         // and store it in s for further tokenization
 450         string s;
 451         char c;
 452         while (is.get(c)) {
 453                 s += c;
 454                 if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
 455                         s = s.substr(0, s.size() - 10);
 456                         break;
 457                 }
 458         }
 459
 460         // tokenize buffer
 461         tokenize(s);
 462 }
 463
 464
 465 void Parser::skipSpaceTokens(istream & is, char c)
 466 {
 467         // skip trailing spaces
 468         while (catcode(c) == catSpace || catcode(c) == catNewline)
 469                 if (!is.get(c))
 470                         break;
 471         //lyxerr << "putting back: " << c << "\n";
 472         is.putback(c);
 473 }
 474
 475
 476 void Parser::tokenize(string const & buffer)
 477 {
 478         static bool init_done = false;
 479
 480         if (!init_done) {
 481                 catInit();
 482                 init_done = true;
 483         }
 484
 485         istringstream is(buffer.c_str(), ios::in | ios::binary);
 486
 487         char c;
 488         while (is.get(c)) {
 489                 //lyxerr << "reading c: " << c << "\n";
 490
 491                 switch (catcode(c)) {
 492                         case catNewline: {
 493                                 ++lineno_;
 494                                 is.get(c);
 495                                 if (catcode(c) == catNewline)
 496                                         ; //push_back(Token("par"));
 497                                 else {
 498                                         push_back(Token(' ', catSpace));
 499                                         is.putback(c);
 500                                 }
 501                                 break;
 502                         }
 503
 504                         case catComment: {
 505                                 while (is.get(c) && catcode(c) != catNewline)
 506                                         ;
 507                                 ++lineno_;
 508                                 break;
 509                         }
 510
 511                         case catEscape: {
 512                                 is.get(c);
 513                                 if (!is) {
 514                                         error("unexpected end of input");
 515                                 } else {
 516                                         string s(1, c);
 517                                         if (catcode(c) == catLetter) {
 518                                                 // collect letters
 519                                                 while (is.get(c) && catcode(c) == catLetter)
 520                                                         s += c;
 521                                                 skipSpaceTokens(is, c);
 522                                         }
 523                                         push_back(Token(s));
 524                                 }
 525                                 break;
 526                         }
 527
 528                         case catSuper:
 529                         case catSub: {
 530                                 push_back(Token(c, catcode(c)));
 531                                 is.get(c);
 532                                 skipSpaceTokens(is, c);
 533                                 break;
 534                         }
 535
 536                         case catIgnore: {
 537                                 lyxerr << "ignoring a char: " << int(c) << "\n";
 538                                 break;
 539                         }
 540
 541                         default:
 542                                 push_back(Token(c, catcode(c)));
 543                 }
 544         }
 545
 546 #ifdef FILEDEBUG
 547         dump();
 548 #endif
 549 }
 550
 551
 552 void Parser::dump() const
 553 {
 554         lyxerr << "\nTokens: ";
 555         for (unsigned i = 0; i < tokens_.size(); ++i) {
 556                 if (i == pos_)
 557                         lyxerr << " <#> ";
 558                 lyxerr << tokens_[i];
 559         }
 560         lyxerr << "\n";
 561 }
 562
 563
 564 void Parser::error(string const & msg)
 565 {
 566         lyxerr << "Line ~" << lineno_ << ": Math parse error: " << msg << endl;
 567         dump();
 568         //exit(1);
 569 }
 570
 571
 572
 573 bool Parser::parse_lines(MathAtom & t, bool numbered, bool outmost)
 574 {
 575         MathGridInset * p = t->asGridInset();
 576         if (!p) {
 577                 dump();
 578                 lyxerr << "error in Parser::parse_lines() 1\n";
 579                 return false;
 580         }
 581
 582         // save global variables
 583         bool   const saved_num   = curr_num_;
 584         string const saved_label = curr_label_;
 585
 586         // read initial hlines
 587         p->rowinfo(0).lines_ = readHLines();
 588
 589         for (int row = 0; true; ++row) {
 590                 // reset global variables
 591                 curr_num_   = numbered;
 592                 curr_label_.erase();
 593
 594                 // reading a row
 595                 for (MathInset::col_type col = 0; true; ++col) {
 596                         //lyxerr << "reading cell " << row << " " << col << " "
 597                         // << p->ncols() << "\n";
 598                         //lyxerr << "ncols: " << p->ncols() << "\n";
 599
 600                         if (col >= p->ncols()) {
 601                                 //lyxerr << "adding col " << col << "\n";
 602                                 p->addCol(p->ncols());
 603                         }
 604
 605                         MathArray & ar = p->cell(col + row * p->ncols());
 606                         parse_into(ar, FLAG_BLOCK);
 607                         // remove 'unnecessary' braces:
 608                         if (ar.size() == 1 && ar.back()->asBraceInset())
 609                                 ar = ar.back()->asBraceInset()->cell(0);
 610                         //lyxerr << "ar: " << ar << "\n";
 611
 612                         // break if cell is not followed by an ampersand
 613                         if (nextToken().cat() != catAlign) {
 614                                 //lyxerr << "less cells read than normal in row/col: "
 615                                 //      << row << " " << col << "\n";
 616                                 break;
 617                         }
 618
 619                         // skip the ampersand
 620                         getToken();
 621                 }
 622
 623                 if (outmost) {
 624                         MathHullInset * m = t->asHullInset();
 625                         if (!m) {
 626                                 lyxerr << "error in Parser::parse_lines() 2\n";
 627                                 return false;
 628                         }
 629                         m->numbered(row, curr_num_);
 630                         m->label(row, curr_label_);
 631                         if (curr_skip_.size()) {
 632                                 m->vcrskip(LyXLength(curr_skip_), row);
 633                                 curr_skip_.erase();
 634                         }
 635                 }
 636
 637                 // is a \\ coming?
 638                 if (nextToken().isCR()) {
 639                         // skip the cr-token
 640                         getToken();
 641
 642                         // try to read a length
 643                         //get
 644
 645                         // read hlines for next row
 646                         p->rowinfo(row + 1).lines_ = readHLines();
 647                 }
 648
 649                 // we are finished if the next token is an 'end'
 650                 if (nextToken().cs() == "end") {
 651                         // skip the end-token
 652                         getToken();
 653                         getArg('{','}');
 654
 655                         // leave the 'read a line'-loop
 656                         break;
 657                 }
 658
 659                 // otherwise, we have to start a new row
 660                 p->appendRow();
 661         }
 662
 663         // restore "global" variables
 664         curr_num_   = saved_num;
 665         curr_label_ = saved_label;
 666
 667         return true;
 668 }
 669
 670
 671 bool Parser::parse_lines2(MathAtom & t, bool braced)
 672 {
 673         MathGridInset * p = t->asGridInset();
 674         if (!p) {
 675                 lyxerr << "error in Parser::parse_lines() 1\n";
 676                 return false;
 677         }
 678
 679         for (int row = 0; true; ++row) {
 680                 // reading a row
 681                 for (MathInset::col_type col = 0; true; ++col) {
 682                         //lyxerr << "reading cell " << row << " " << col << " " << p->ncols() << "\n";
 683
 684                         if (col >= p->ncols()) {
 685                                 //lyxerr << "adding col " << col << "\n";
 686                                 p->addCol(p->ncols());
 687                         }
 688
 689                         parse_into(p->cell(col + row * p->ncols()), FLAG_BLOCK2);
 690                         //lyxerr << "read cell: " << p->cell(col + row * p->ncols()) << "\n";
 691
 692                         // break if cell is not followed by an ampersand
 693                         if (nextToken().cat() != catAlign) {
 694                                 //lyxerr << "less cells read than normal in row/col: " << row << " " << col << "\n";
 695                                 break;
 696                         }
 697
 698                         // skip the ampersand
 699                         getToken();
 700                 }
 701
 702                 // is a \\ coming?
 703                 if (nextToken().isCR()) {
 704                         // skip the cr-token
 705                         getToken();
 706                 }
 707
 708                 // we are finished if the next token is the one we expected
 709                 // skip the end-token
 710                 // leave the 'read a line'-loop
 711                 if (braced) {
 712                         if (nextToken().cat() == catEnd) {
 713                                 getToken();
 714                                 break;
 715                         }
 716                 } else {
 717                         if (nextToken().cs() == "end") {
 718                                 getToken();
 719                                 getArg('{','}');
 720                                 break;
 721                         }
 722                 }
 723
 724                 // otherwise, we have to start a new row
 725                 p->appendRow();
 726         }
 727
 728         return true;
 729 }
 730
 731
 732
 733
 734 bool Parser::parse_macro(string & name)
 735 {
 736         int nargs = 0;
 737         name = "{error}";
 738         skipSpaces();
 739
 740         if (nextToken().cs() == "def") {
 741
 742                 getToken();
 743                 name = getToken().cs();
 744
 745                 string pars;
 746                 while (good() && nextToken().cat() != catBegin)
 747                         pars += getToken().cs();
 748
 749                 if (!good()) {
 750                         lyxerr << "bad stream in parse_macro\n";
 751                         dump();
 752                         return false;
 753                 }
 754
 755                 //lyxerr << "read \\def parameter list '" << pars << "'\n";
 756                 if (!pars.empty()) {
 757                         lyxerr << "can't handle non-empty parameter lists\n";
 758                         dump();
 759                         return false;
 760                 }
 761
 762         } else if (nextToken().cs() == "newcommand") {
 763
 764                 getToken();
 765
 766                 if (getToken().cat() != catBegin) {
 767                         lyxerr << "'{' in \\newcommand expected (1) \n";
 768                         dump();
 769                         return false;
 770                 }
 771
 772                 name = getToken().cs();
 773
 774                 if (getToken().cat() != catEnd) {
 775                         lyxerr << "'}' expected\n";
 776                         return false;
 777                 }
 778
 779                 string arg  = getArg('[', ']');
 780                 if (!arg.empty())
 781                         nargs = atoi(arg.c_str());
 782
 783         } else {
 784                 lyxerr << "\\newcommand or \\def  expected\n";
 785                 return false;
 786         }
 787
 788
 789         if (getToken().cat() != catBegin) {
 790                 lyxerr << "'{' in macro definition expected (2)\n";
 791                 return false;
 792         }
 793
 794         MathArray ar1;
 795         parse_into(ar1, FLAG_BRACE_LAST);
 796
 797         // we cannot handle recursive stuff at all
 798         MathArray test;
 799         test.push_back(createMathInset(name));
 800         if (ar1.contains(test)) {
 801                 lyxerr << "we cannot handle recursive macros at all.\n";
 802                 return false;
 803         }
 804
 805         // is a version for display attached?
 806         MathArray ar2;
 807         parse_into(ar2, FLAG_ITEM);
 808
 809         MathMacroTable::create(name, nargs, ar1, ar2);
 810         return true;
 811 }
 812
 813
 814 bool Parser::parse_normal(MathAtom & matrix)
 815 {
 816         skipSpaces();
 817         Token const & t = getToken();
 818
 819         if (t.cs() == "(") {
 820                 matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 821                 parse_into(matrix->cell(0), 0);
 822                 return true;
 823         }
 824
 825         if (t.cat() == catMath) {
 826                 Token const & n = getToken();
 827                 if (n.cat() == catMath) {
 828                         // TeX's $$...$$ syntax for displayed math
 829                         matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 830                         MathHullInset * p = matrix->asHullInset();
 831                         parse_into(p->cell(0), 0);
 832                         p->numbered(0, curr_num_);
 833                         p->label(0, curr_label_);
 834                 } else {
 835                         // simple $...$  stuff
 836                         putback();
 837                         matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 838                         parse_into(matrix->cell(0), 0);
 839                 }
 840                 return true;
 841         }
 842
 843         if (!t.cs().size()) {
 844                 lyxerr << "start of math expected, got '" << t << "'\n";
 845                 return false;
 846         }
 847
 848         string const & cs = t.cs();
 849
 850         if (cs == "[") {
 851                 curr_num_ = 0;
 852                 curr_label_.erase();
 853                 matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 854                 MathHullInset * p = matrix->asHullInset();
 855                 parse_into(p->cell(0), 0);
 856                 p->numbered(0, curr_num_);
 857                 p->label(0, curr_label_);
 858                 return true;
 859         }
 860
 861         if (cs != "begin") {
 862                 lyxerr[Debug::MATHED]
 863                         << "'begin' of un-simple math expected, got '" << cs << "'\n";
 864                 return false;
 865         }
 866
 867         string const name = getArg('{', '}');
 868
 869         if (name == "math") {
 870                 matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
 871                 parse_into(matrix->cell(0), 0);
 872                 return true;
 873         }
 874
 875         if (name == "equation" || name == "equation*" || name == "displaymath") {
 876                 curr_num_ = (name == "equation");
 877                 curr_label_.erase();
 878                 matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
 879                 MathHullInset * p = matrix->asHullInset();
 880                 parse_into(p->cell(0), FLAG_END);
 881                 p->numbered(0, curr_num_);
 882                 p->label(0, curr_label_);
 883                 return true;
 884         }
 885
 886         if (name == "eqnarray" || name == "eqnarray*") {
 887                 matrix = MathAtom(new MathHullInset(LM_OT_EQNARRAY));
 888                 return parse_lines(matrix, !stared(name), true);
 889         }
 890
 891         if (name == "align" || name == "align*") {
 892                 matrix = MathAtom(new MathHullInset(LM_OT_ALIGN));
 893                 return parse_lines(matrix, !stared(name), true);
 894         }
 895
 896         if (name == "alignat" || name == "alignat*") {
 897                 int nc = 2 * atoi(getArg('{', '}').c_str());
 898                 matrix = MathAtom(new MathHullInset(LM_OT_ALIGNAT, nc));
 899                 return parse_lines(matrix, !stared(name), true);
 900         }
 901
 902         if (name == "xalignat" || name == "xalignat*") {
 903                 int nc = 2 * atoi(getArg('{', '}').c_str());
 904                 matrix = MathAtom(new MathHullInset(LM_OT_XALIGNAT, nc));
 905                 return parse_lines(matrix, !stared(name), true);
 906         }
 907
 908         if (name == "xxalignat") {
 909                 int nc = 2 * atoi(getArg('{', '}').c_str());
 910                 matrix = MathAtom(new MathHullInset(LM_OT_XXALIGNAT, nc));
 911                 return parse_lines(matrix, !stared(name), true);
 912         }
 913
 914         if (name == "multline" || name == "multline*") {
 915                 matrix = MathAtom(new MathHullInset(LM_OT_MULTLINE));
 916                 return parse_lines(matrix, !stared(name), true);
 917         }
 918
 919         if (name == "gather" || name == "gather*") {
 920                 matrix = MathAtom(new MathHullInset(LM_OT_GATHER));
 921                 return parse_lines(matrix, !stared(name), true);
 922         }
 923
 924         lyxerr[Debug::MATHED] << "1: unknown math environment: " << name << "\n";
 925         lyxerr << "1: unknown math environment: " << name << "\n";
 926         return false;
 927 }
 928
 929
 930 void Parser::parse_into(MathArray & array, unsigned flags, MathTextCodes code)
 931 {
 932         parse_into1(array, flags, code);
 933         // remove 'unnecessary' braces:
 934         if (array.size() == 1 && array.back()->asBraceInset()) {
 935                 lyxerr << "extra braces removed\n";
 936                 array = array.back()->asBraceInset()->cell(0);
 937         }
 938 }
 939
 940
 941 void Parser::parse_into1(MathArray & array, unsigned flags, MathTextCodes code)
 942 {
 943         bool panic  = false;
 944         int  limits = 0;
 945
 946         while (good()) {
 947                 Token const & t = getToken();
 948
 949 #ifdef FILEDEBUG
 950                 lyxerr << "t: " << t << " flags: " << flags << "\n";
 951                 //array.dump();
 952                 lyxerr << "\n";
 953 #endif
 954
 955                 if (flags & FLAG_ITEM) {
 956                         if (t.cat() == catSpace)
 957                                 continue;
 958
 959                         flags &= ~FLAG_ITEM;
 960                         if (t.cat() == catBegin) {
 961                                 // skip the brace and collect everything to the next matching
 962                                 // closing brace
 963                                 flags |= FLAG_BRACE_LAST;
 964                                 continue;
 965                         }
 966
 967                         // handle only this single token, leave the loop if done
 968                         flags |= FLAG_LEAVE;
 969                 }
 970
 971                 if (flags & FLAG_BLOCK) {
 972                         if (t.cat() == catAlign || t.isCR() || t.cs() == "end") {
 973                                 putback();
 974                                 return;
 975                         }
 976                 }
 977
 978                 if (flags & FLAG_BLOCK2) {
 979                         if (t.cat() == catAlign || t.isCR() || t.cs() == "end"
 980                                         || t.cat() == catEnd) {
 981                                 putback();
 982                                 return;
 983                         }
 984                 }
 985
 986                 //
 987                 // cat codes
 988                 //
 989                 if (t.cat() == catMath) {
 990                         if (flags & FLAG_BOX) {
 991                                 // we are inside an mbox, so opening new math is allowed
 992                                 array.push_back(MathAtom(new MathHullInset(LM_OT_SIMPLE)));
 993                                 parse_into(array.back()->cell(0), 0);
 994                         } else {
 995                                 // otherwise this is the end of the formula
 996                                 break;
 997                         }
 998                 }
 999
1000                 else if (t.cat() == catLetter)
1001                         add(array, t.character(), code);
1002
1003                 else if (t.cat() == catSpace && code == LM_TC_TEXTRM)
1004                         add(array, t.character(), code);
1005
1006                 else if (t.cat() == catParameter) {
1007                         Token const & n = getToken();
1008                         array.push_back(MathAtom(new MathMacroArgument(n.character()-'0', code)));
1009                 }
1010
1011                 else if (t.cat() == catBegin) {
1012                         MathArray ar;
1013                         parse_into(ar, FLAG_BRACE_LAST);
1014 #ifndef WITH_WARNINGS
1015 #warning this might be wrong in general!
1016 #endif
1017                         // ignore braces around simple items
1018                         if ((ar.size() == 1 && !ar.front()->needsBraces()
1019        || (ar.size() == 2 && !ar.front()->needsBraces()
1020                                             && ar.back()->asScriptInset()))
1021        || (ar.size() == 0 && array.size() == 0))
1022                         {
1023                                 array.push_back(ar);
1024                         } else {
1025                                 array.push_back(MathAtom(new MathBraceInset));
1026                                 array.back()->cell(0).swap(ar);
1027                         }
1028                 }
1029
1030                 else if (t.cat() == catEnd) {
1031                         if (flags & FLAG_BRACE_LAST)
1032                                 return;
1033                         dump();
1034                         lyxerr << "found '}' unexpectedly, array: '" << array << "'\n";
1035                         //lyxerr << "found '}' unexpectedly\n";
1036                         //lyx::Assert(0);
1037                         //add(array, '}', LM_TC_TEX);
1038                 }
1039
1040                 else if (t.cat() == catAlign) {
1041                         lyxerr << "found tab unexpectedly, array: '" << array << "'\n";
1042                         //lyxerr << "found tab unexpectedly\n";
1043                         add(array, '&', LM_TC_TEX);
1044                 }
1045
1046                 else if (t.cat() == catSuper || t.cat() == catSub) {
1047                         bool up = (t.cat() == catSuper);
1048                         MathScriptInset * p = 0;
1049                         if (array.size())
1050                                 p = array.back()->asScriptInset();
1051                         if (!p || p->has(up)) {
1052                                 array.push_back(MathAtom(new MathScriptInset(up)));
1053                                 p = array.back()->asScriptInset();
1054                         }
1055                         p->ensure(up);
1056                         parse_into(p->cell(up), FLAG_ITEM);
1057                         p->limits(limits);
1058                         limits = 0;
1059                 }
1060
1061                 else if (t.character() == ']' && (flags & FLAG_BRACK_END))
1062                         return;
1063
1064                 else if (t.cat() == catOther)
1065                         add(array, t.character(), code);
1066
1067                 //
1068                 // control sequences
1069                 //
1070                 else if (t.cs() == "protect")
1071                         // ignore \\protect, will be re-added during output
1072                         ;
1073
1074                 else if (t.cs() == "end")
1075                         break;
1076
1077                 else if (t.cs() == ")")
1078                         break;
1079
1080                 else if (t.cs() == "]")
1081                         break;
1082
1083                 else if (t.cs() == "\\") {
1084                         curr_skip_ = getArg('[', ']');
1085                         //lyxerr << "found newline unexpectedly, array: '" << array << "'\n";
1086                         lyxerr << "found newline unexpectedly\n";
1087                         array.push_back(createMathInset("\\"));
1088                 }
1089
1090                 else if (t.cs() == "limits")
1091                         limits = 1;
1092
1093                 else if (t.cs() == "nolimits")
1094                         limits = -1;
1095
1096                 else if (t.cs() == "nonumber")
1097                         curr_num_ = false;
1098
1099                 else if (t.cs() == "number")
1100                         curr_num_ = true;
1101
1102                 else if (t.cs() == "sqrt") {
1103                         char c = getChar();
1104                         if (c == '[') {
1105                                 array.push_back(MathAtom(new MathRootInset));
1106                                 parse_into(array.back()->cell(0), FLAG_BRACK_END);
1107                                 parse_into(array.back()->cell(1), FLAG_ITEM);
1108                         } else {
1109                                 putback();
1110                                 array.push_back(MathAtom(new MathSqrtInset));
1111                                 parse_into(array.back()->cell(0), FLAG_ITEM);
1112                         }
1113                 }
1114
1115                 else if (t.cs() == "left") {
1116                         string l = getToken().asString();
1117                         MathArray ar;
1118                         parse_into(ar, FLAG_RIGHT);
1119                         string r = getToken().asString();
1120                         MathAtom dl(new MathDelimInset(l, r));
1121                         dl->cell(0) = ar;
1122                         array.push_back(dl);
1123                 }
1124
1125                 else if (t.cs() == "right") {
1126                         if (!(flags & FLAG_RIGHT)) {
1127                                 //lyxerr << "got so far: '" << array << "'\n";
1128                                 error("Unmatched right delimiter");
1129                         }
1130                         return;
1131                 }
1132
1133                 else if (t.cs() == "begin") {
1134                         string const name = getArg('{', '}');
1135                         if (name == "array" || name == "subarray") {
1136                                 string const valign = getArg('[', ']') + 'c';
1137                                 string const halign = getArg('{', '}');
1138                                 array.push_back(MathAtom(new MathArrayInset(name, valign[0], halign)));
1139                                 parse_lines(array.back(), false, false);
1140                         } else if (name == "split" || name == "cases" ||
1141                                          name == "gathered" || name == "aligned") {
1142                                 array.push_back(createMathInset(name));
1143                                 parse_lines(array.back(), false, false);
1144                         } else if (name == "matrix"  || name == "pmatrix" || name == "bmatrix" ||
1145                                          name == "vmatrix" || name == "Vmatrix") {
1146                                 array.push_back(createMathInset(name));
1147                                 parse_lines2(array.back(), false);
1148                         } else
1149                                 lyxerr << "unknow math inset begin '" << name << "'\n";
1150                 }
1151
1152                 else if (t.cs() == "kern") {
1153 #ifdef WITH_WARNINGS
1154 #warning A hack...
1155 #endif
1156                         string s;
1157                         while (1) {
1158                                 Token const & t = getToken();
1159                                 if (!good()) {
1160                                         putback();
1161                                         break;
1162                                 }
1163                                 s += t.character();
1164                                 if (isValidLength(s))
1165                                         break;
1166                         }
1167                         array.push_back(MathAtom(new MathKernInset(s)));
1168                 }
1169
1170 /*
1171                 else if (t.cs() == "lyxkern") {
1172                         MathAtom p = createMathInset(t.cs());
1173                         parse_into(p->cell(0), flags, code);
1174                         array.push_back(p);
1175                 }
1176 */
1177
1178                 else if (t.cs() == "label") {
1179                         curr_label_ = getArg('{', '}');
1180                 }
1181
1182                 else if (t.cs() == "choose" || t.cs() == "over" || t.cs() == "atop") {
1183                         MathAtom p = createMathInset(t.cs());
1184                         array.swap(p->cell(0));
1185                         parse_into(p->cell(1), flags, code);
1186                         array.push_back(p);
1187                         return;
1188                 }
1189
1190                 else if (t.cs() == "substack") {
1191                         array.push_back(createMathInset(t.cs()));
1192                         skipBegin();
1193                         parse_lines2(array.back(), true);
1194                 }
1195
1196                 else if (t.cs() == "xymatrix") {
1197                         array.push_back(createMathInset(t.cs()));
1198                         skipBegin();
1199                         parse_lines2(array.back(), true);
1200                 }
1201
1202 #if 0
1203                 // Disabled
1204                 else if (1 && t.cs() == "ar") {
1205                         MathXYArrowInset * p = new MathXYArrowInset;
1206
1207                         // try to read target
1208                         char c = getChar();
1209                         if (c == '[') {
1210                                 parse_into(p->cell(0), FLAG_BRACK_END);
1211                                 //lyxerr << "read target: " << p->cell(0) << "\n";
1212                         } else {
1213                                 putback();
1214                         }
1215
1216                         // try to read label
1217                         if (nextToken().cat() == catSuper || nextToken().cat() == catSub) {
1218                                 p->up_ = nextToken().cat() == catSuper;
1219                                 getToken();
1220                                 parse_into(p->cell(1), FLAG_ITEM);
1221                                 //lyxerr << "read label: " << p->cell(1) << "\n";
1222                         }
1223
1224                         array.push_back(MathAtom(p));
1225                         //lyxerr << "read array: " << array << "\n";
1226                 }
1227 #endif
1228
1229 #if 0
1230                 else if (t.cs() == "mbox" || t.cs() == "text") {
1231                         //array.push_back(createMathInset(t.cs()));
1232                         array.push_back(MathAtom(new MathBoxInset(t.cs())));
1233                         // slurp in the argument of mbox
1234
1235                         MathBoxInset * p = array.back()->asBoxInset();
1236                         //lyx::assert(p);
1237                 }
1238 #endif
1239
1240
1241                 else if (t.cs().size()) {
1242                         latexkeys const * l = in_word_set(t.cs());
1243                         if (l) {
1244                                 if (l->token == LM_TK_FONT) {
1245                                         //lyxerr << "starting font\n";
1246                                         //CatCode catSpaceSave = theCatcode[' '];
1247                                         //if (l->id == LM_TC_TEXTRM) {
1248                                         //      // temporarily change catcode
1249                                         //      theCatcode[' '] = catLetter;
1250                                         //}
1251
1252                                         MathArray ar;
1253                                         parse_into(ar, FLAG_ITEM, static_cast<MathTextCodes>(l->id));
1254                                         array.push_back(ar);
1255
1256                                         // undo catcode changes
1257                                         ////theCatcode[' '] = catSpaceSave;
1258                                         //lyxerr << "ending font\n";
1259                                 }
1260
1261                                 else if (l->token == LM_TK_OLDFONT) {
1262                                         code = static_cast<MathTextCodes>(l->id);
1263                                 }
1264
1265                                 else if (l->token == LM_TK_BOX) {
1266                                         MathAtom p = createMathInset(t.cs());
1267                                         parse_into(p->cell(0), FLAG_ITEM | FLAG_BOX, LM_TC_BOX);
1268                                         array.push_back(p);
1269                                 }
1270
1271                                 else if (l->token == LM_TK_STY) {
1272                                         MathAtom p = createMathInset(t.cs());
1273                                         parse_into(p->cell(0), flags, code);
1274                                         array.push_back(p);
1275                                         return;
1276                                 }
1277
1278                                 else {
1279                                         MathAtom p = createMathInset(t.cs());
1280                                         for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
1281                                                 parse_into(p->cell(i), FLAG_ITEM);
1282                                         array.push_back(p);
1283                                 }
1284                         }
1285
1286                         else {
1287                                 MathAtom p = createMathInset(t.cs());
1288                                 for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
1289                                         parse_into(p->cell(i), FLAG_ITEM);
1290                                 array.push_back(p);
1291                         }
1292                 }
1293
1294
1295                 if (flags & FLAG_LEAVE) {
1296                         flags &= ~FLAG_LEAVE;
1297                         break;
1298                 }
1299         }
1300
1301         if (panic) {
1302                 lyxerr << " Math Panic, expect problems!\n";
1303                 //   Search for the end command.
1304                 Token t;
1305                 do {
1306                         t = getToken();
1307                 } while (good() && t.cs() != "end");
1308         }
1309 }
1310
1311
1312
1313 } // anonymous namespace
1314
1315
1316 void mathed_parse_cell(MathArray & ar, string const & str)
1317 {
1318         istringstream is(str.c_str());
1319         mathed_parse_cell(ar, is);
1320 }
1321
1322
1323 void mathed_parse_cell(MathArray & ar, istream & is)
1324 {
1325         Parser(is).parse_into(ar, 0);
1326 }
1327
1328
1329
1330 bool mathed_parse_macro(string & name, string const & str)
1331 {
1332         istringstream is(str.c_str());
1333         Parser parser(is);
1334         return parser.parse_macro(name);
1335 }
1336
1337 bool mathed_parse_macro(string & name, istream & is)
1338 {
1339         Parser parser(is);
1340         return parser.parse_macro(name);
1341 }
1342
1343 bool mathed_parse_macro(string & name, LyXLex & lex)
1344 {
1345         Parser parser(lex);
1346         return parser.parse_macro(name);
1347 }
1348
1349
1350
1351 bool mathed_parse_normal(MathAtom & t, string const & str)
1352 {
1353         istringstream is(str.c_str());
1354         Parser parser(is);
1355         return parser.parse_normal(t);
1356 }
1357
1358 bool mathed_parse_normal(MathAtom & t, istream & is)
1359 {
1360         Parser parser(is);
1361         return parser.parse_normal(t);
1362 }
1363
1364 bool mathed_parse_normal(MathAtom & t, LyXLex & lex)
1365 {
1366         Parser parser(lex);
1367         return parser.parse_normal(t);
1368 }