src/mathed/math_parser.C

   1 /**
   2  * \file math_parser.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  */
  10
  11 /*
  12
  13 If someone desperately needs partial "structures" (such as a few
  14 cells of an array inset or similar) (s)he could uses the
  15 following hack as starting point to write some macros:
  16
  17   \newif\ifcomment
  18   \commentfalse
  19   \ifcomment
  20           \def\makeamptab{\catcode`\&=4\relax}
  21           \def\makeampletter{\catcode`\&=11\relax}
  22     \def\b{\makeampletter\expandafter\makeamptab\bi}
  23     \long\def\bi#1\e{}
  24   \else
  25     \def\b{}\def\e{}
  26   \fi
  27
  28   ...
  29
  30   \[\begin{array}{ccc}
  31 1
  32 &
  33
  34   \end{array}\]
  35
  36 */
  37
  38
  39 #include <config.h>
  40
  41 #include "math_parser.h"
  42 #include "math_arrayinset.h"
  43 #include "math_braceinset.h"
  44 #include "math_charinset.h"
  45 #include "math_commentinset.h"
  46 #include "math_deliminset.h"
  47 #include "math_envinset.h"
  48 #include "math_factory.h"
  49 #include "math_kerninset.h"
  50 #include "math_macro.h"
  51 #include "math_macroarg.h"
  52 #include "math_macrotemplate.h"
  53 #include "math_parinset.h"
  54 #include "math_rootinset.h"
  55 #include "math_scriptinset.h"
  56 #include "math_sqrtinset.h"
  57 #include "math_support.h"
  58 #include "math_tabularinset.h"
  59
  60 //#include "insets/insetref.h"
  61 #include "ref_inset.h"
  62
  63 #include "lyxlex.h"
  64 #include "debug.h"
  65
  66 #include "support/convert.h"
  67
  68 #include <sstream>
  69
  70 using std::endl;
  71 using std::fill;
  72
  73 using std::string;
  74 using std::ios;
  75 using std::istream;
  76 using std::istringstream;
  77 using std::ostream;
  78 using std::vector;
  79
  80
  81 //#define FILEDEBUG
  82
  83
  84 namespace {
  85
  86 MathInset::mode_type asMode(MathInset::mode_type oldmode, string const & str)
  87 {
  88         //lyxerr << "handling mode: '" << str << "'" << endl;
  89         if (str == "mathmode")
  90                 return MathInset::MATH_MODE;
  91         if (str == "textmode" || str == "forcetext")
  92                 return MathInset::TEXT_MODE;
  93         return oldmode;
  94 }
  95
  96
  97 bool stared(string const & s)
  98 {
  99         string::size_type const n = s.size();
 100         return n && s[n - 1] == '*';
 101 }
 102
 103
 104 /*!
 105  * Add the row \p cellrow to \p grid.
 106  * \returns wether the row could be added. Adding a row can fail for
 107  * environments like "equation" that have a fixed number of rows.
 108  */
 109 bool addRow(MathGridInset & grid, MathGridInset::row_type & cellrow,
 110             string const & vskip)
 111 {
 112         ++cellrow;
 113         if (cellrow == grid.nrows()) {
 114                 //lyxerr << "adding row " << cellrow << endl;
 115                 grid.addRow(cellrow - 1);
 116                 if (cellrow == grid.nrows()) {
 117                         // We can't add a row to this grid, so let's
 118                         // append the content of this cell to the previous
 119                         // one.
 120                         // This does not happen in well formed .lyx files,
 121                         // but LyX versions 1.3.x and older could create
 122                         // such files and tex2lyx can still do that.
 123                         --cellrow;
 124                         lyxerr << "ignoring extra row";
 125                         if (!vskip.empty())
 126                                 lyxerr << " with extra space " << vskip;
 127                         lyxerr << '.' << endl;
 128                         return false;
 129                 }
 130         }
 131         grid.vcrskip(LyXLength(vskip), cellrow - 1);
 132         return true;
 133 }
 134
 135
 136 /*!
 137  * Add the column \p cellcol to \p grid.
 138  * \returns wether the column could be added. Adding a column can fail for
 139  * environments like "eqnarray" that have a fixed number of columns.
 140  */
 141 bool addCol(MathGridInset & grid, MathGridInset::col_type & cellcol)
 142 {
 143         ++cellcol;
 144         if (cellcol == grid.ncols()) {
 145                 //lyxerr << "adding column " << cellcol << endl;
 146                 grid.addCol(cellcol - 1);
 147                 if (cellcol == grid.ncols()) {
 148                         // We can't add a column to this grid, so let's
 149                         // append the content of this cell to the previous
 150                         // one.
 151                         // This does not happen in well formed .lyx files,
 152                         // but LyX versions 1.3.x and older could create
 153                         // such files and tex2lyx can still do that.
 154                         --cellcol;
 155                         lyxerr << "ignoring extra column." << endl;
 156                         return false;
 157                 }
 158         }
 159         return true;
 160 }
 161
 162
 163 // These are TeX's catcodes
 164 enum CatCode {
 165         catEscape,     // 0    backslash
 166         catBegin,      // 1    {
 167         catEnd,        // 2    }
 168         catMath,       // 3    $
 169         catAlign,      // 4    &
 170         catNewline,    // 5    ^^M
 171         catParameter,  // 6    #
 172         catSuper,      // 7    ^
 173         catSub,        // 8    _
 174         catIgnore,     // 9
 175         catSpace,      // 10   space
 176         catLetter,     // 11   a-zA-Z
 177         catOther,      // 12   none of the above
 178         catActive,     // 13   ~
 179         catComment,    // 14   %
 180         catInvalid     // 15   <delete>
 181 };
 182
 183 CatCode theCatcode[256];
 184
 185
 186 inline CatCode catcode(unsigned char c)
 187 {
 188         return theCatcode[c];
 189 }
 190
 191
 192 enum {
 193         FLAG_ALIGN      = 1 << 0,  //  next & or \\ ends the parsing process
 194         FLAG_BRACE_LAST = 1 << 1,  //  next closing brace ends the parsing
 195         FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
 196         FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
 197         FLAG_BRACK_LAST = 1 << 4,  //  next closing bracket ends the parsing
 198         FLAG_TEXTMODE   = 1 << 5,  //  we are in a box
 199         FLAG_ITEM       = 1 << 6,  //  read a (possibly braced token)
 200         FLAG_LEAVE      = 1 << 7,  //  leave the loop at the end
 201         FLAG_SIMPLE     = 1 << 8,  //  next $ leaves the loop
 202         FLAG_EQUATION   = 1 << 9,  //  next \] leaves the loop
 203         FLAG_SIMPLE2    = 1 << 10, //  next \) leaves the loop
 204         FLAG_OPTION     = 1 << 11, //  read [...] style option
 205         FLAG_BRACED     = 1 << 12  //  read {...} style argument
 206 };
 207
 208
 209 //
 210 // Helper class for parsing
 211 //
 212
 213 class Token {
 214 public:
 215         ///
 216         Token() : cs_(), char_(0), cat_(catIgnore) {}
 217         ///
 218         Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
 219         ///
 220         Token(string const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
 221
 222         ///
 223         string const & cs() const { return cs_; }
 224         ///
 225         CatCode cat() const { return cat_; }
 226         ///
 227         char character() const { return char_; }
 228         ///
 229         string asString() const { return cs_.size() ? cs_ : string(1, char_); }
 230
 231 private:
 232         ///
 233         string cs_;
 234         ///
 235         char char_;
 236         ///
 237         CatCode cat_;
 238 };
 239
 240 ostream & operator<<(ostream & os, Token const & t)
 241 {
 242         if (t.cs().size())
 243                 os << '\\' << t.cs();
 244         else if (t.cat() == catLetter)
 245                 os << t.character();
 246         else
 247                 os << '[' << t.character() << ',' << t.cat() << ']';
 248         return os;
 249 }
 250
 251
 252 class Parser {
 253 public:
 254         ///
 255         typedef  MathInset::mode_type mode_type;
 256
 257         ///
 258         Parser(LyXLex & lex);
 259         ///
 260         Parser(istream & is);
 261
 262         ///
 263         bool parse(MathAtom & at);
 264         ///
 265         void parse(MathArray & array, unsigned flags, mode_type mode);
 266         ///
 267         void parse1(MathGridInset & grid, unsigned flags, mode_type mode,
 268                 bool numbered);
 269         ///
 270         MathArray parse(unsigned flags, mode_type mode);
 271         ///
 272         int lineno() const { return lineno_; }
 273         ///
 274         void putback();
 275
 276 private:
 277         ///
 278         void parse2(MathAtom & at, unsigned flags, mode_type mode, bool numbered);
 279         /// get arg delimited by 'left' and 'right'
 280         string getArg(char left, char right);
 281         ///
 282         char getChar();
 283         ///
 284         void error(string const & msg);
 285         /// dump contents to screen
 286         void dump() const;
 287         ///
 288         void tokenize(istream & is);
 289         ///
 290         void tokenize(string const & s);
 291         ///
 292         void skipSpaceTokens(istream & is, char c);
 293         ///
 294         void push_back(Token const & t);
 295         ///
 296         void pop_back();
 297         ///
 298         Token const & prevToken() const;
 299         ///
 300         Token const & nextToken() const;
 301         ///
 302         Token const & getToken();
 303         /// skips spaces if any
 304         void skipSpaces();
 305         ///
 306         void lex(string const & s);
 307         ///
 308         bool good() const;
 309         ///
 310         string parse_verbatim_item();
 311         ///
 312         string parse_verbatim_option();
 313
 314         ///
 315         int lineno_;
 316         ///
 317         vector<Token> tokens_;
 318         ///
 319         unsigned pos_;
 320 };
 321
 322
 323 Parser::Parser(LyXLex & lexer)
 324         : lineno_(lexer.getLineNo()), pos_(0)
 325 {
 326         tokenize(lexer.getStream());
 327         lexer.eatLine();
 328 }
 329
 330
 331 Parser::Parser(istream & is)
 332         : lineno_(0), pos_(0)
 333 {
 334         tokenize(is);
 335 }
 336
 337
 338 void Parser::push_back(Token const & t)
 339 {
 340         tokens_.push_back(t);
 341 }
 342
 343
 344 void Parser::pop_back()
 345 {
 346         tokens_.pop_back();
 347 }
 348
 349
 350 Token const & Parser::prevToken() const
 351 {
 352         static const Token dummy;
 353         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 354 }
 355
 356
 357 Token const & Parser::nextToken() const
 358 {
 359         static const Token dummy;
 360         return good() ? tokens_[pos_] : dummy;
 361 }
 362
 363
 364 Token const & Parser::getToken()
 365 {
 366         static const Token dummy;
 367         //lyxerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << endl;
 368         return good() ? tokens_[pos_++] : dummy;
 369 }
 370
 371
 372 void Parser::skipSpaces()
 373 {
 374         while (nextToken().cat() == catSpace || nextToken().cat() == catNewline)
 375                 getToken();
 376 }
 377
 378
 379 void Parser::putback()
 380 {
 381         --pos_;
 382 }
 383
 384
 385 bool Parser::good() const
 386 {
 387         return pos_ < tokens_.size();
 388 }
 389
 390
 391 char Parser::getChar()
 392 {
 393         if (!good())
 394                 error("The input stream is not well...");
 395         return tokens_[pos_++].character();
 396 }
 397
 398
 399 string Parser::getArg(char left, char right)
 400 {
 401         skipSpaces();
 402
 403         string result;
 404         char c = getChar();
 405
 406         if (c != left)
 407                 putback();
 408         else
 409                 while ((c = getChar()) != right && good())
 410                         result += c;
 411
 412         return result;
 413 }
 414
 415
 416 void Parser::skipSpaceTokens(istream & is, char c)
 417 {
 418         // skip trailing spaces
 419         while (catcode(c) == catSpace || catcode(c) == catNewline)
 420                 if (!is.get(c))
 421                         break;
 422         //lyxerr << "putting back: " << c << endl;
 423         is.putback(c);
 424 }
 425
 426
 427 void Parser::tokenize(istream & is)
 428 {
 429         // eat everything up to the next \end_inset or end of stream
 430         // and store it in s for further tokenization
 431         string s;
 432         char c;
 433         while (is.get(c)) {
 434                 s += c;
 435                 if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
 436                         s = s.substr(0, s.size() - 10);
 437                         break;
 438                 }
 439         }
 440         // Remove the space after \end_inset
 441         if (is.get(c) && c != ' ')
 442                 is.unget();
 443
 444         // tokenize buffer
 445         tokenize(s);
 446 }
 447
 448
 449 void Parser::tokenize(string const & buffer)
 450 {
 451         istringstream is(buffer, ios::in | ios::binary);
 452
 453         char c;
 454         while (is.get(c)) {
 455                 //lyxerr << "reading c: " << c << endl;
 456
 457                 switch (catcode(c)) {
 458                         case catNewline: {
 459                                 ++lineno_;
 460                                 is.get(c);
 461                                 if (catcode(c) == catNewline)
 462                                         ; //push_back(Token("par"));
 463                                 else {
 464                                         push_back(Token('\n', catNewline));
 465                                         is.putback(c);
 466                                 }
 467                                 break;
 468                         }
 469
 470 /*
 471                         case catComment: {
 472                                 while (is.get(c) && catcode(c) != catNewline)
 473                                         ;
 474                                 ++lineno_;
 475                                 break;
 476                         }
 477 */
 478
 479                         case catEscape: {
 480                                 is.get(c);
 481                                 if (!is) {
 482                                         error("unexpected end of input");
 483                                 } else {
 484                                         string s(1, c);
 485                                         if (catcode(c) == catLetter) {
 486                                                 // collect letters
 487                                                 while (is.get(c) && catcode(c) == catLetter)
 488                                                         s += c;
 489                                                 skipSpaceTokens(is, c);
 490                                         }
 491                                         push_back(Token(s));
 492                                 }
 493                                 break;
 494                         }
 495
 496                         case catSuper:
 497                         case catSub: {
 498                                 push_back(Token(c, catcode(c)));
 499                                 is.get(c);
 500                                 skipSpaceTokens(is, c);
 501                                 break;
 502                         }
 503
 504                         case catIgnore: {
 505                                 lyxerr << "ignoring a char: " << int(c) << endl;
 506                                 break;
 507                         }
 508
 509                         default:
 510                                 push_back(Token(c, catcode(c)));
 511                 }
 512         }
 513
 514 #ifdef FILEDEBUG
 515         dump();
 516 #endif
 517 }
 518
 519
 520 void Parser::dump() const
 521 {
 522         lyxerr << "\nTokens: ";
 523         for (unsigned i = 0; i < tokens_.size(); ++i) {
 524                 if (i == pos_)
 525                         lyxerr << " <#> ";
 526                 lyxerr << tokens_[i];
 527         }
 528         lyxerr << " pos: " << pos_ << endl;
 529 }
 530
 531
 532 void Parser::error(string const & msg)
 533 {
 534         lyxerr << "Line ~" << lineno_ << ": Math parse error: " << msg << endl;
 535         dump();
 536         //exit(1);
 537 }
 538
 539
 540 bool Parser::parse(MathAtom & at)
 541 {
 542         skipSpaces();
 543         MathArray ar;
 544         parse(ar, false, MathInset::UNDECIDED_MODE);
 545         if (ar.size() != 1 || ar.front()->getType() == "none") {
 546                 lyxerr << "unusual contents found: " << ar << endl;
 547                 at = MathAtom(new MathParInset(ar));
 548                 //if (at->nargs() > 0)
 549                 //      at.nucleus()->cell(0) = ar;
 550                 //else
 551                 //      lyxerr << "unusual contents found: " << ar << endl;
 552                 return true;
 553         }
 554         at = ar[0];
 555         return true;
 556 }
 557
 558
 559 string Parser::parse_verbatim_option()
 560 {
 561         skipSpaces();
 562         string res;
 563         if (nextToken().character() == '[') {
 564                 Token t = getToken();
 565                 for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
 566                         if (t.cat() == catBegin) {
 567                                 putback();
 568                                 res += '{' + parse_verbatim_item() + '}';
 569                         } else
 570                                 res += t.asString();
 571                 }
 572         }
 573         return res;
 574 }
 575
 576
 577 string Parser::parse_verbatim_item()
 578 {
 579         skipSpaces();
 580         string res;
 581         if (nextToken().cat() == catBegin) {
 582                 Token t = getToken();
 583                 for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
 584                         if (t.cat() == catBegin) {
 585                                 putback();
 586                                 res += '{' + parse_verbatim_item() + '}';
 587                         }
 588                         else
 589                                 res += t.asString();
 590                 }
 591         }
 592         return res;
 593 }
 594
 595
 596 MathArray Parser::parse(unsigned flags, mode_type mode)
 597 {
 598         MathArray ar;
 599         parse(ar, flags, mode);
 600         return ar;
 601 }
 602
 603
 604 void Parser::parse(MathArray & array, unsigned flags, mode_type mode)
 605 {
 606         MathGridInset grid(1, 1);
 607         parse1(grid, flags, mode, false);
 608         array = grid.cell(0);
 609 }
 610
 611
 612 void Parser::parse2(MathAtom & at, const unsigned flags, const mode_type mode,
 613         const bool numbered)
 614 {
 615         parse1(*(at.nucleus()->asGridInset()), flags, mode, numbered);
 616 }
 617
 618
 619 void Parser::parse1(MathGridInset & grid, unsigned flags,
 620         const mode_type mode, const bool numbered)
 621 {
 622         int limits = 0;
 623         MathGridInset::row_type cellrow = 0;
 624         MathGridInset::col_type cellcol = 0;
 625         MathArray * cell = &grid.cell(grid.index(cellrow, cellcol));
 626
 627         if (grid.asHullInset())
 628                 grid.asHullInset()->numbered(cellrow, numbered);
 629
 630         //dump();
 631         //lyxerr << " flags: " << flags << endl;
 632         //lyxerr << " mode: " << mode  << endl;
 633         //lyxerr << "grid: " << grid << endl;
 634
 635         while (good()) {
 636                 Token const & t = getToken();
 637
 638 #ifdef FILEDEBUG
 639                 lyxerr << "t: " << t << " flags: " << flags << endl;
 640                 lyxerr << "mode: " << mode  << endl;
 641                 cell->dump();
 642                 lyxerr << endl;
 643 #endif
 644
 645                 if (flags & FLAG_ITEM) {
 646
 647                         if (t.cat() == catBegin) {
 648                                 // skip the brace and collect everything to the next matching
 649                                 // closing brace
 650                                 parse1(grid, FLAG_BRACE_LAST, mode, numbered);
 651                                 return;
 652                         }
 653
 654                         // handle only this single token, leave the loop if done
 655                         flags = FLAG_LEAVE;
 656                 }
 657
 658
 659                 if (flags & FLAG_BRACED) {
 660                         if (t.cat() == catSpace)
 661                                 continue;
 662
 663                         if (t.cat() != catBegin) {
 664                                 error("opening brace expected");
 665                                 return;
 666                         }
 667
 668                         // skip the brace and collect everything to the next matching
 669                         // closing brace
 670                         flags = FLAG_BRACE_LAST;
 671                 }
 672
 673
 674                 if (flags & FLAG_OPTION) {
 675                         if (t.cat() == catOther && t.character() == '[') {
 676                                 MathArray ar;
 677                                 parse(ar, FLAG_BRACK_LAST, mode);
 678                                 cell->append(ar);
 679                         } else {
 680                                 // no option found, put back token and we are done
 681                                 putback();
 682                         }
 683                         return;
 684                 }
 685
 686                 //
 687                 // cat codes
 688                 //
 689                 if (t.cat() == catMath) {
 690                         if (mode != MathInset::MATH_MODE) {
 691                                 // we are inside some text mode thingy, so opening new math is allowed
 692                                 Token const & n = getToken();
 693                                 if (n.cat() == catMath) {
 694                                         // TeX's $$...$$ syntax for displayed math
 695                                         cell->push_back(MathAtom(new MathHullInset("equation")));
 696                                         parse2(cell->back(), FLAG_SIMPLE, MathInset::MATH_MODE, false);
 697                                         getToken(); // skip the second '$' token
 698                                 } else {
 699                                         // simple $...$  stuff
 700                                         putback();
 701                                         cell->push_back(MathAtom(new MathHullInset("simple")));
 702                                         parse2(cell->back(), FLAG_SIMPLE, MathInset::MATH_MODE, false);
 703                                 }
 704                         }
 705
 706                         else if (flags & FLAG_SIMPLE) {
 707                                 // this is the end of the formula
 708                                 return;
 709                         }
 710
 711                         else {
 712                                 error("something strange in the parser");
 713                                 break;
 714                         }
 715                 }
 716
 717                 else if (t.cat() == catLetter)
 718                         cell->push_back(MathAtom(new MathCharInset(t.character())));
 719
 720                 else if (t.cat() == catSpace && mode != MathInset::MATH_MODE) {
 721                         if (cell->empty() || cell->back()->getChar() != ' ')
 722                                 cell->push_back(MathAtom(new MathCharInset(t.character())));
 723                 }
 724
 725                 else if (t.cat() == catNewline && mode != MathInset::MATH_MODE) {
 726                         if (cell->empty() || cell->back()->getChar() != ' ')
 727                                 cell->push_back(MathAtom(new MathCharInset(' ')));
 728                 }
 729
 730                 else if (t.cat() == catParameter) {
 731                         Token const & n = getToken();
 732                         cell->push_back(MathAtom(new MathMacroArgument(n.character()-'0')));
 733                 }
 734
 735                 else if (t.cat() == catActive)
 736                         cell->push_back(MathAtom(new MathCharInset(t.character())));
 737
 738                 else if (t.cat() == catBegin) {
 739                         MathArray ar;
 740                         parse(ar, FLAG_BRACE_LAST, mode);
 741                         // do not create a BraceInset if they were written by LyX
 742                         // this helps to keep the annoyance of  "a choose b"  to a minimum
 743                         if (ar.size() == 1 && ar[0]->extraBraces())
 744                                 cell->append(ar);
 745                         else
 746                                 cell->push_back(MathAtom(new MathBraceInset(ar)));
 747                 }
 748
 749                 else if (t.cat() == catEnd) {
 750                         if (flags & FLAG_BRACE_LAST)
 751                                 return;
 752                         error("found '}' unexpectedly");
 753                         //BOOST_ASSERT(false);
 754                         //add(cell, '}', LM_TC_TEX);
 755                 }
 756
 757                 else if (t.cat() == catAlign) {
 758                         //lyxerr << " column now " << (cellcol + 1)
 759                         //       << " max: " << grid.ncols() << endl;
 760                         if (flags & FLAG_ALIGN)
 761                                 return;
 762                         if (addCol(grid, cellcol))
 763                                 cell = &grid.cell(grid.index(cellrow, cellcol));
 764                 }
 765
 766                 else if (t.cat() == catSuper || t.cat() == catSub) {
 767                         bool up = (t.cat() == catSuper);
 768                         // we need no new script inset if the last thing was a scriptinset,
 769                         // which has that script already not the same script already
 770                         if (!cell->size())
 771                                 cell->push_back(MathAtom(new MathScriptInset(up)));
 772                         else if (cell->back()->asScriptInset() &&
 773                                         !cell->back()->asScriptInset()->has(up))
 774                                 cell->back().nucleus()->asScriptInset()->ensure(up);
 775                         else if (cell->back()->asScriptInset())
 776                                 cell->push_back(MathAtom(new MathScriptInset(up)));
 777                         else
 778                                 cell->back() = MathAtom(new MathScriptInset(cell->back(), up));
 779                         MathScriptInset * p = cell->back().nucleus()->asScriptInset();
 780                         // special handling of {}-bases
 781                         // is this always correct?
 782                         // It appears that this is wrong (Dekel)
 783                         //if (p->nuc().size() == 1 && p->nuc().back()->asNestInset() &&
 784                         //    p->nuc().back()->extraBraces())
 785                         //      p->nuc() = p->nuc().back()->asNestInset()->cell(0);
 786                         parse(p->cell(p->idxOfScript(up)), FLAG_ITEM, mode);
 787                         if (limits) {
 788                                 p->limits(limits);
 789                                 limits = 0;
 790                         }
 791                 }
 792
 793                 else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
 794                         //lyxerr << "finished reading option" << endl;
 795                         return;
 796                 }
 797
 798                 else if (t.cat() == catOther)
 799                         cell->push_back(MathAtom(new MathCharInset(t.character())));
 800
 801                 else if (t.cat() == catComment) {
 802                         string s;
 803                         while (good()) {
 804                                 Token const & t = getToken();
 805                                 if (t.cat() == catNewline)
 806                                         break;
 807                                 s += t.asString();
 808                         }
 809                         cell->push_back(MathAtom(new MathCommentInset(s)));
 810                         skipSpaces();
 811                 }
 812
 813                 //
 814                 // control sequences
 815                 //
 816
 817                 else if (t.cs() == "lyxlock") {
 818                         if (cell->size())
 819                                 cell->back().nucleus()->lock(true);
 820                 }
 821
 822                 else if (t.cs() == "def" ||
 823                         t.cs() == "newcommand" ||
 824                         t.cs() == "renewcommand")
 825                 {
 826                         string const type = t.cs();
 827                         string name;
 828                         int nargs = 0;
 829                         if (t.cs() == "def") {
 830                                 // get name
 831                                 name = getToken().cs();
 832
 833                                 // read parameter
 834                                 string pars;
 835                                 while (good() && nextToken().cat() != catBegin) {
 836                                         pars += getToken().cs();
 837                                         ++nargs;
 838                                 }
 839                                 nargs /= 2;
 840                                 //lyxerr << "read \\def parameter list '" << pars << "'" << endl;
 841
 842                         } else { // t.cs() == "newcommand" || t.cs() == "renewcommand"
 843
 844                                 if (getToken().cat() != catBegin) {
 845                                         error("'{' in \\newcommand expected (1) ");
 846                                         return;
 847                                 }
 848
 849                                 name = getToken().cs();
 850
 851                                 if (getToken().cat() != catEnd) {
 852                                         error("'}' in \\newcommand expected");
 853                                         return;
 854                                 }
 855
 856                                 string const arg  = getArg('[', ']');
 857                                 if (!arg.empty())
 858                                         nargs = convert<int>(arg);
 859
 860                         }
 861
 862                         MathArray ar1;
 863                         parse(ar1, FLAG_ITEM, MathInset::UNDECIDED_MODE);
 864
 865                         // we cannot handle recursive stuff at all
 866                         //MathArray test;
 867                         //test.push_back(createMathInset(name));
 868                         //if (ar1.contains(test)) {
 869                         //      error("we cannot handle recursive macros at all.");
 870                         //      return;
 871                         //}
 872
 873                         // is a version for display attached?
 874                         skipSpaces();
 875                         MathArray ar2;
 876                         if (nextToken().cat() == catBegin)
 877                                 parse(ar2, FLAG_ITEM, MathInset::MATH_MODE);
 878
 879                         cell->push_back(MathAtom(new MathMacroTemplate(name, nargs, type,
 880                                 ar1, ar2)));
 881                 }
 882
 883                 else if (t.cs() == "(") {
 884                         cell->push_back(MathAtom(new MathHullInset("simple")));
 885                         parse2(cell->back(), FLAG_SIMPLE2, MathInset::MATH_MODE, false);
 886                 }
 887
 888                 else if (t.cs() == "[") {
 889                         cell->push_back(MathAtom(new MathHullInset("equation")));
 890                         parse2(cell->back(), FLAG_EQUATION, MathInset::MATH_MODE, false);
 891                 }
 892
 893                 else if (t.cs() == "protect")
 894                         // ignore \\protect, will hopefully be re-added during output
 895                         ;
 896
 897                 else if (t.cs() == "end") {
 898                         if (flags & FLAG_END) {
 899                                 // eat environment name
 900                                 //string const name =
 901                                 getArg('{', '}');
 902                                 // FIXME: check that we ended the correct environment
 903                                 return;
 904                         }
 905                         error("found 'end' unexpectedly");
 906                 }
 907
 908                 else if (t.cs() == ")") {
 909                         if (flags & FLAG_SIMPLE2)
 910                                 return;
 911                         error("found '\\)' unexpectedly");
 912                 }
 913
 914                 else if (t.cs() == "]") {
 915                         if (flags & FLAG_EQUATION)
 916                                 return;
 917                         error("found '\\]' unexpectedly");
 918                 }
 919
 920                 else if (t.cs() == "\\") {
 921                         if (flags & FLAG_ALIGN)
 922                                 return;
 923                         if (addRow(grid, cellrow, getArg('[', ']'))) {
 924                                 cellcol = 0;
 925                                 if (grid.asHullInset())
 926                                         grid.asHullInset()->numbered(
 927                                                         cellrow, numbered);
 928                                 cell = &grid.cell(grid.index(cellrow,
 929                                                              cellcol));
 930                         }
 931                 }
 932
 933 #if 0
 934                 else if (t.cs() == "multicolumn") {
 935                         // extract column count and insert dummy cells
 936                         MathArray count;
 937                         parse(count, FLAG_ITEM, mode);
 938                         int cols = 1;
 939                         if (!extractNumber(count, cols)) {
 940                                 lyxerr << " can't extract number of cells from " << count << endl;
 941                         }
 942                         // resize the table if necessary
 943                         for (int i = 0; i < cols; ++i) {
 944                                 if (addCol(grid, cellcol)) {
 945                                         cell = &grid.cell(grid.index(
 946                                                         cellrow, cellcol));
 947                                         // mark this as dummy
 948                                         grid.cellinfo(grid.index(
 949                                                 cellrow, cellcol)).dummy_ = true;
 950                                 }
 951                         }
 952                         // the last cell is the real thing, not a dummy
 953                         grid.cellinfo(grid.index(cellrow, cellcol)).dummy_ = false;
 954
 955                         // read special alignment
 956                         MathArray align;
 957                         parse(align, FLAG_ITEM, mode);
 958                         //grid.cellinfo(grid.index(cellrow, cellcol)).align_ = extractString(align);
 959
 960                         // parse the remaining contents into the "real" cell
 961                         parse(*cell, FLAG_ITEM, mode);
 962                 }
 963 #endif
 964
 965                 else if (t.cs() == "limits")
 966                         limits = 1;
 967
 968                 else if (t.cs() == "nolimits")
 969                         limits = -1;
 970
 971                 else if (t.cs() == "nonumber") {
 972                         if (grid.asHullInset())
 973                                 grid.asHullInset()->numbered(cellrow, false);
 974                 }
 975
 976                 else if (t.cs() == "number") {
 977                         if (grid.asHullInset())
 978                                 grid.asHullInset()->numbered(cellrow, true);
 979                 }
 980
 981                 else if (t.cs() == "hline") {
 982                         grid.rowinfo(cellrow).lines_ ++;
 983                 }
 984
 985                 else if (t.cs() == "sqrt") {
 986                         MathArray ar;
 987                         parse(ar, FLAG_OPTION, mode);
 988                         if (ar.size()) {
 989                                 cell->push_back(MathAtom(new MathRootInset));
 990                                 cell->back().nucleus()->cell(0) = ar;
 991                                 parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
 992                         } else {
 993                                 cell->push_back(MathAtom(new MathSqrtInset));
 994                                 parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
 995                         }
 996                 }
 997
 998                 else if (t.cs() == "xrightarrow" || t.cs() == "xleftarrow") {
 999                         cell->push_back(createMathInset(t.cs()));
1000                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, mode);
1001                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1002                 }
1003
1004                 else if (t.cs() == "ref" || t.cs() == "prettyref" ||
1005                                 t.cs() == "pageref" || t.cs() == "vpageref" || t.cs() == "vref") {
1006                         cell->push_back(MathAtom(new RefInset(t.cs())));
1007                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, mode);
1008                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1009                 }
1010
1011                 else if (t.cs() == "left") {
1012                         skipSpaces();
1013                         string l = getToken().asString();
1014                         MathArray ar;
1015                         parse(ar, FLAG_RIGHT, mode);
1016                         skipSpaces();
1017                         string r = getToken().asString();
1018                         cell->push_back(MathAtom(new MathDelimInset(l, r, ar)));
1019                 }
1020
1021                 else if (t.cs() == "right") {
1022                         if (flags & FLAG_RIGHT)
1023                                 return;
1024                         //lyxerr << "got so far: '" << cell << "'" << endl;
1025                         error("Unmatched right delimiter");
1026                         return;
1027                 }
1028
1029                 else if (t.cs() == "begin") {
1030                         string const name = getArg('{', '}');
1031
1032                         if (name == "array" || name == "subarray") {
1033                                 string const valign = parse_verbatim_option() + 'c';
1034                                 string const halign = parse_verbatim_item();
1035                                 cell->push_back(MathAtom(new MathArrayInset(name, valign[0], halign)));
1036                                 parse2(cell->back(), FLAG_END, mode, false);
1037                         }
1038
1039                         else if (name == "tabular") {
1040                                 string const valign = parse_verbatim_option() + 'c';
1041                                 string const halign = parse_verbatim_item();
1042                                 cell->push_back(MathAtom(new MathTabularInset(name, valign[0], halign)));
1043                                 parse2(cell->back(), FLAG_END, MathInset::TEXT_MODE, false);
1044                         }
1045
1046                         else if (name == "split" || name == "cases" ||
1047                                          name == "gathered" || name == "aligned" ||
1048                                    name == "alignedat") {
1049                                 cell->push_back(createMathInset(name));
1050                                 parse2(cell->back(), FLAG_END, mode, false);
1051                         }
1052
1053                         else if (name == "math") {
1054                                 cell->push_back(MathAtom(new MathHullInset("simple")));
1055                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, true);
1056                         }
1057
1058                         else if (name == "equation" || name == "equation*"
1059                                         || name == "displaymath") {
1060                                 cell->push_back(MathAtom(new MathHullInset("equation")));
1061                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, (name == "equation"));
1062                         }
1063
1064                         else if (name == "eqnarray" || name == "eqnarray*") {
1065                                 cell->push_back(MathAtom(new MathHullInset("eqnarray")));
1066                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1067                         }
1068
1069                         else if (name == "align" || name == "align*") {
1070                                 cell->push_back(MathAtom(new MathHullInset("align")));
1071                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1072                         }
1073
1074                         else if (name == "flalign" || name == "flalign*") {
1075                                 cell->push_back(MathAtom(new MathHullInset("flalign")));
1076                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1077                         }
1078
1079                         else if (name == "alignat" || name == "alignat*") {
1080                                 // ignore this for a while
1081                                 getArg('{', '}');
1082                                 cell->push_back(MathAtom(new MathHullInset("alignat")));
1083                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1084                         }
1085
1086                         else if (name == "xalignat" || name == "xalignat*") {
1087                                 // ignore this for a while
1088                                 getArg('{', '}');
1089                                 cell->push_back(MathAtom(new MathHullInset("xalignat")));
1090                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1091                         }
1092
1093                         else if (name == "xxalignat") {
1094                                 // ignore this for a while
1095                                 getArg('{', '}');
1096                                 cell->push_back(MathAtom(new MathHullInset("xxalignat")));
1097                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1098                         }
1099
1100                         else if (name == "multline" || name == "multline*") {
1101                                 cell->push_back(MathAtom(new MathHullInset("multline")));
1102                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1103                         }
1104
1105                         else if (name == "gather" || name == "gather*") {
1106                                 cell->push_back(MathAtom(new MathHullInset("gather")));
1107                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1108                         }
1109
1110                         else if (latexkeys const * l = in_word_set(name)) {
1111                                 if (l->inset == "matrix") {
1112                                         cell->push_back(createMathInset(name));
1113                                         parse2(cell->back(), FLAG_END, mode, false);
1114                                 }
1115                         }
1116
1117                         else {
1118                                 dump();
1119                                 lyxerr << "found unknown math environment '" << name << "'" << endl;
1120                                 // create generic environment inset
1121                                 cell->push_back(MathAtom(new MathEnvInset(name)));
1122                                 parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1123                         }
1124                 }
1125
1126                 else if (t.cs() == "kern") {
1127 #ifdef WITH_WARNINGS
1128 #warning A hack...
1129 #endif
1130                         string s;
1131                         while (true) {
1132                                 Token const & t = getToken();
1133                                 if (!good()) {
1134                                         putback();
1135                                         break;
1136                                 }
1137                                 s += t.character();
1138                                 if (isValidLength(s))
1139                                         break;
1140                         }
1141                         cell->push_back(MathAtom(new MathKernInset(s)));
1142                 }
1143
1144                 else if (t.cs() == "label") {
1145                         string label = parse_verbatim_item();
1146                         MathArray ar;
1147                         asArray(label, ar);
1148                         if (grid.asHullInset()) {
1149                                 grid.asHullInset()->label(cellrow, label);
1150                         } else {
1151                                 cell->push_back(createMathInset(t.cs()));
1152                                 cell->push_back(MathAtom(new MathBraceInset(ar)));
1153                         }
1154                 }
1155
1156                 else if (t.cs() == "choose" || t.cs() == "over" || t.cs() == "atop") {
1157                         MathAtom at = createMathInset(t.cs());
1158                         at.nucleus()->cell(0) = *cell;
1159                         cell->clear();
1160                         parse(at.nucleus()->cell(1), flags, mode);
1161                         cell->push_back(at);
1162                         return;
1163                 }
1164
1165                 else if (t.cs() == "color") {
1166                         MathAtom at = createMathInset(t.cs());
1167                         parse(at.nucleus()->cell(0), FLAG_ITEM, MathInset::TEXT_MODE);
1168                         parse(at.nucleus()->cell(1), flags, mode);
1169                         cell->push_back(at);
1170                         return;
1171                 }
1172
1173                 else if (t.cs() == "substack") {
1174                         cell->push_back(createMathInset(t.cs()));
1175                         parse2(cell->back(), FLAG_ITEM, mode, false);
1176                 }
1177
1178                 else if (t.cs() == "framebox" || t.cs() == "makebox") {
1179                         cell->push_back(createMathInset(t.cs()));
1180                         parse(cell->back().nucleus()->cell(0), FLAG_OPTION, MathInset::TEXT_MODE);
1181                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, MathInset::TEXT_MODE);
1182                         parse(cell->back().nucleus()->cell(2), FLAG_ITEM, MathInset::TEXT_MODE);
1183                 }
1184
1185 #if 0
1186                 else if (t.cs() == "infer") {
1187                         MathArray ar;
1188                         parse(ar, FLAG_OPTION, mode);
1189                         cell->push_back(createMathInset(t.cs()));
1190                         parse2(cell->back(), FLAG_ITEM, mode, false);
1191                 }
1192
1193                 // Disabled
1194                 else if (1 && t.cs() == "ar") {
1195                         auto_ptr<MathXYArrowInset> p(new MathXYArrowInset);
1196                         // try to read target
1197                         parse(p->cell(0), FLAG_OTPTION, mode);
1198                         // try to read label
1199                         if (nextToken().cat() == catSuper || nextToken().cat() == catSub) {
1200                                 p->up_ = nextToken().cat() == catSuper;
1201                                 getToken();
1202                                 parse(p->cell(1), FLAG_ITEM, mode);
1203                                 //lyxerr << "read label: " << p->cell(1) << endl;
1204                         }
1205
1206                         cell->push_back(MathAtom(p.release()));
1207                         //lyxerr << "read cell: " << cell << endl;
1208                 }
1209 #endif
1210
1211                 else if (t.cs().size()) {
1212                         latexkeys const * l = in_word_set(t.cs());
1213                         if (l) {
1214                                 if (l->inset == "font") {
1215                                         cell->push_back(createMathInset(t.cs()));
1216                                         parse(cell->back().nucleus()->cell(0),
1217                                                 FLAG_ITEM, asMode(mode, l->extra));
1218                                 }
1219
1220                                 else if (l->inset == "oldfont") {
1221                                         cell->push_back(createMathInset(t.cs()));
1222                                         parse(cell->back().nucleus()->cell(0),
1223                                                 flags | FLAG_ALIGN, asMode(mode, l->extra));
1224                                         if (prevToken().cat() != catAlign &&
1225                                             prevToken().cs() != "\\")
1226                                                 return;
1227                                         putback();
1228                                 }
1229
1230                                 else if (l->inset == "style") {
1231                                         cell->push_back(createMathInset(t.cs()));
1232                                         parse(cell->back().nucleus()->cell(0),
1233                                                 flags | FLAG_ALIGN, mode);
1234                                         if (prevToken().cat() != catAlign &&
1235                                             prevToken().cs() != "\\")
1236                                                 return;
1237                                         putback();
1238                                 }
1239
1240                                 else {
1241                                         MathAtom at = createMathInset(t.cs());
1242                                         for (MathInset::idx_type i = 0; i < at->nargs(); ++i)
1243                                                 parse(at.nucleus()->cell(i),
1244                                                         FLAG_ITEM, asMode(mode, l->extra));
1245                                         cell->push_back(at);
1246                                 }
1247                         }
1248
1249                         else {
1250                                 MathAtom at = createMathInset(t.cs());
1251                                 MathInset::mode_type m = mode;
1252                                 //if (m == MathInset::UNDECIDED_MODE)
1253                                 //lyxerr << "default creation: m1: " << m << endl;
1254                                 if (at->currentMode() != MathInset::UNDECIDED_MODE)
1255                                         m = at->currentMode();
1256                                 //lyxerr << "default creation: m2: " << m << endl;
1257                                 MathInset::idx_type start = 0;
1258                                 // this fails on \bigg[...\bigg]
1259                                 //MathArray opt;
1260                                 //parse(opt, FLAG_OPTION, MathInset::VERBATIM_MODE);
1261                                 //if (opt.size()) {
1262                                 //      start = 1;
1263                                 //      at.nucleus()->cell(0) = opt;
1264                                 //}
1265                                 for (MathInset::idx_type i = start; i < at->nargs(); ++i) {
1266                                         parse(at.nucleus()->cell(i), FLAG_ITEM, m);
1267                                         skipSpaces();
1268                                 }
1269                                 cell->push_back(at);
1270                         }
1271                 }
1272
1273
1274                 if (flags & FLAG_LEAVE) {
1275                         flags &= ~FLAG_LEAVE;
1276                         break;
1277                 }
1278         }
1279 }
1280
1281
1282
1283 } // anonymous namespace
1284
1285
1286 void mathed_parse_cell(MathArray & ar, string const & str)
1287 {
1288         istringstream is(str);
1289         mathed_parse_cell(ar, is);
1290 }
1291
1292
1293 void mathed_parse_cell(MathArray & ar, istream & is)
1294 {
1295         Parser(is).parse(ar, 0, MathInset::MATH_MODE);
1296 }
1297
1298
1299 bool mathed_parse_normal(MathAtom & t, string const & str)
1300 {
1301         istringstream is(str);
1302         return Parser(is).parse(t);
1303 }
1304
1305
1306 bool mathed_parse_normal(MathAtom & t, istream & is)
1307 {
1308         return Parser(is).parse(t);
1309 }
1310
1311
1312 bool mathed_parse_normal(MathAtom & t, LyXLex & lex)
1313 {
1314         return Parser(lex).parse(t);
1315 }
1316
1317
1318 void mathed_parse_normal(MathGridInset & grid, string const & str)
1319 {
1320         istringstream is(str);
1321         Parser(is).parse1(grid, 0, MathInset::MATH_MODE, false);
1322 }
1323
1324
1325 void initParser()
1326 {
1327         fill(theCatcode, theCatcode + 256, catOther);
1328         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
1329         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
1330
1331         theCatcode[int('\\')] = catEscape;
1332         theCatcode[int('{')]  = catBegin;
1333         theCatcode[int('}')]  = catEnd;
1334         theCatcode[int('$')]  = catMath;
1335         theCatcode[int('&')]  = catAlign;
1336         theCatcode[int('\n')] = catNewline;
1337         theCatcode[int('#')]  = catParameter;
1338         theCatcode[int('^')]  = catSuper;
1339         theCatcode[int('_')]  = catSub;
1340         theCatcode[int(0x7f)] = catIgnore;
1341         theCatcode[int(' ')]  = catSpace;
1342         theCatcode[int('\t')] = catSpace;
1343         theCatcode[int('\r')] = catNewline;
1344         theCatcode[int('~')]  = catActive;
1345         theCatcode[int('%')]  = catComment;
1346 }