src/mathed/MathParser.cpp

   1 /**
   2  * \file MathParser.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  */
  10
  11 /*
  12
  13 If someone desperately needs partial "structures" (such as a few
  14 cells of an array inset or similar) (s)he could uses the
  15 following hack as starting point to write some macros:
  16
  17   \newif\ifcomment
  18   \commentfalse
  19   \ifcomment
  20           \def\makeamptab{\catcode`\&=4\relax}
  21           \def\makeampletter{\catcode`\&=11\relax}
  22     \def\b{\makeampletter\expandafter\makeamptab\bi}
  23     \long\def\bi#1\e{}
  24   \else
  25     \def\b{}\def\e{}
  26   \fi
  27   ...
  28
  29   \[\begin{array}{ccc}
  30 1
  31 &
  32
  33   \end{array}\]
  34
  35 */
  36
  37
  38 #include <config.h>
  39
  40 #include "MathParser.h"
  41
  42 #include "InsetMathArray.h"
  43 #include "InsetMathBig.h"
  44 #include "InsetMathBrace.h"
  45 #include "InsetMathChar.h"
  46 #include "InsetMathColor.h"
  47 #include "InsetMathComment.h"
  48 #include "InsetMathDelim.h"
  49 #include "InsetMathEnv.h"
  50 #include "InsetMathFrac.h"
  51 #include "InsetMathKern.h"
  52 #include "MathMacro.h"
  53 #include "InsetMathPar.h"
  54 #include "InsetMathRef.h"
  55 #include "InsetMathRoot.h"
  56 #include "InsetMathScript.h"
  57 #include "InsetMathSplit.h"
  58 #include "InsetMathSqrt.h"
  59 #include "InsetMathTabular.h"
  60 #include "MathMacroTemplate.h"
  61 #include "MathFactory.h"
  62 #include "MathMacroArgument.h"
  63 #include "MathSupport.h"
  64
  65 #include "Encoding.h"
  66 #include "Lexer.h"
  67
  68 #include "support/debug.h"
  69 #include "support/convert.h"
  70 #include "support/docstream.h"
  71 #include "support/lstrings.h"
  72
  73 #include <sstream>
  74
  75 //#define FILEDEBUG
  76
  77 using namespace std;
  78
  79 namespace lyx {
  80
  81 using support::subst;
  82
  83 namespace {
  84
  85 InsetMath::mode_type asMode(InsetMath::mode_type oldmode, docstring const & str)
  86 {
  87         //lyxerr << "handling mode: '" << str << "'" << endl;
  88         if (str == "mathmode")
  89                 return InsetMath::MATH_MODE;
  90         if (str == "textmode" || str == "forcetext")
  91                 return InsetMath::TEXT_MODE;
  92         return oldmode;
  93 }
  94
  95
  96 bool stared(docstring const & s)
  97 {
  98         size_t const n = s.size();
  99         return n && s[n - 1] == '*';
 100 }
 101
 102
 103 docstring escapeSpecialChars(docstring const & str)
 104 {
 105         return subst(subst(subst(subst(subst(subst(subst(subst(subst(str,
 106                         from_ascii("\\"), from_ascii("\\backslash ")),
 107                         from_ascii("^"), from_ascii("\\mathcircumflex ")),
 108                         from_ascii("_"), from_ascii("\\_")),
 109                         from_ascii("$"), from_ascii("\\$")),
 110                         from_ascii("#"), from_ascii("\\#")),
 111                         from_ascii("&"), from_ascii("\\&")),
 112                         from_ascii("%"), from_ascii("\\%")),
 113                         from_ascii("{"), from_ascii("\\{")),
 114                         from_ascii("}"), from_ascii("\\}"));
 115 }
 116
 117
 118 /*!
 119  * Add the row \p cellrow to \p grid.
 120  * \returns wether the row could be added. Adding a row can fail for
 121  * environments like "equation" that have a fixed number of rows.
 122  */
 123 bool addRow(InsetMathGrid & grid, InsetMathGrid::row_type & cellrow,
 124             docstring const & vskip, bool allow_newpage_ = true)
 125 {
 126         ++cellrow;
 127         if (cellrow == grid.nrows()) {
 128                 //lyxerr << "adding row " << cellrow << endl;
 129                 grid.addRow(cellrow - 1);
 130                 if (cellrow == grid.nrows()) {
 131                         // We can't add a row to this grid, so let's
 132                         // append the content of this cell to the previous
 133                         // one.
 134                         // This does not happen in well formed .lyx files,
 135                         // but LyX versions 1.3.x and older could create
 136                         // such files and tex2lyx can still do that.
 137                         --cellrow;
 138                         lyxerr << "ignoring extra row";
 139                         if (!vskip.empty())
 140                                 lyxerr << " with extra space " << to_utf8(vskip);
 141                         if (!allow_newpage_)
 142                                 lyxerr << " with no page break allowed";
 143                         lyxerr << '.' << endl;
 144                         return false;
 145                 }
 146         }
 147         grid.vcrskip(Length(to_utf8(vskip)), cellrow - 1);
 148         grid.rowinfo(cellrow - 1).allow_newpage_ = allow_newpage_;
 149         return true;
 150 }
 151
 152
 153 /*!
 154  * Add the column \p cellcol to \p grid.
 155  * \returns wether the column could be added. Adding a column can fail for
 156  * environments like "eqnarray" that have a fixed number of columns.
 157  */
 158 bool addCol(InsetMathGrid & grid, InsetMathGrid::col_type & cellcol)
 159 {
 160         ++cellcol;
 161         if (cellcol == grid.ncols()) {
 162                 //lyxerr << "adding column " << cellcol << endl;
 163                 grid.addCol(cellcol);
 164                 if (cellcol == grid.ncols()) {
 165                         // We can't add a column to this grid, so let's
 166                         // append the content of this cell to the previous
 167                         // one.
 168                         // This does not happen in well formed .lyx files,
 169                         // but LyX versions 1.3.x and older could create
 170                         // such files and tex2lyx can still do that.
 171                         --cellcol;
 172                         lyxerr << "ignoring extra column." << endl;
 173                         return false;
 174                 }
 175         }
 176         return true;
 177 }
 178
 179
 180 /*!
 181  * Check wether the last row is empty and remove it if yes.
 182  * Otherwise the following code
 183  * \verbatim
 184 \begin{array}{|c|c|}
 185 \hline
 186 1 & 2 \\ \hline
 187 3 & 4 \\ \hline
 188 \end{array}
 189  * \endverbatim
 190  * will result in a grid with 3 rows (+ the dummy row that is always present),
 191  * because the last '\\' opens a new row.
 192  */
 193 void delEmptyLastRow(InsetMathGrid & grid)
 194 {
 195         InsetMathGrid::row_type const row = grid.nrows() - 1;
 196         for (InsetMathGrid::col_type col = 0; col < grid.ncols(); ++col) {
 197                 if (!grid.cell(grid.index(row, col)).empty())
 198                         return;
 199         }
 200         // Copy the row information of the empty row (which would contain the
 201         // last hline in the example above) to the dummy row and delete the
 202         // empty row.
 203         grid.rowinfo(row + 1) = grid.rowinfo(row);
 204         grid.delRow(row);
 205 }
 206
 207
 208 // These are TeX's catcodes
 209 enum CatCode {
 210         catEscape,     // 0    backslash
 211         catBegin,      // 1    {
 212         catEnd,        // 2    }
 213         catMath,       // 3    $
 214         catAlign,      // 4    &
 215         catNewline,    // 5    ^^M
 216         catParameter,  // 6    #
 217         catSuper,      // 7    ^
 218         catSub,        // 8    _
 219         catIgnore,     // 9
 220         catSpace,      // 10   space
 221         catLetter,     // 11   a-zA-Z
 222         catOther,      // 12   none of the above
 223         catActive,     // 13   ~
 224         catComment,    // 14   %
 225         catInvalid     // 15   <delete>
 226 };
 227
 228 CatCode theCatcode[128];
 229
 230
 231 inline CatCode catcode(char_type c)
 232 {
 233         /* The only characters that are not catOther lie in the pure ASCII
 234          * range. Therefore theCatcode has only 128 entries.
 235          * TeX itself deals with 8bit characters, so if needed this table
 236          * could be enlarged to 256 entries.
 237          * Any larger value does not make sense, since the fact that we use
 238          * unicode internally does not change Knuth's TeX engine.
 239          * Apart from that a table for the full 21bit UCS4 range would waste
 240          * too much memory. */
 241         if (c >= 128)
 242                 return catOther;
 243
 244         return theCatcode[c];
 245 }
 246
 247
 248 enum {
 249         FLAG_ALIGN      = 1 << 0,  //  next & or \\ ends the parsing process
 250         FLAG_BRACE_LAST = 1 << 1,  //  next closing brace ends the parsing
 251         FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
 252         FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
 253         FLAG_BRACK_LAST = 1 << 4,  //  next closing bracket ends the parsing
 254         FLAG_TEXTMODE   = 1 << 5,  //  we are in a box
 255         FLAG_ITEM       = 1 << 6,  //  read a (possibly braced) token
 256         FLAG_LEAVE      = 1 << 7,  //  leave the loop at the end
 257         FLAG_SIMPLE     = 1 << 8,  //  next $ leaves the loop
 258         FLAG_EQUATION   = 1 << 9,  //  next \] leaves the loop
 259         FLAG_SIMPLE2    = 1 << 10, //  next \) leaves the loop
 260         FLAG_OPTION     = 1 << 11, //  read [...] style option
 261         FLAG_BRACED     = 1 << 12  //  read {...} style argument
 262 };
 263
 264
 265 //
 266 // Helper class for parsing
 267 //
 268
 269 class Token {
 270 public:
 271         ///
 272         Token() : cs_(), char_(0), cat_(catIgnore) {}
 273         ///
 274         Token(char_type c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
 275         ///
 276         explicit Token(docstring const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
 277
 278         ///
 279         docstring const & cs() const { return cs_; }
 280         ///
 281         CatCode cat() const { return cat_; }
 282         ///
 283         char_type character() const { return char_; }
 284         ///
 285         docstring asString() const { return cs_.size() ? cs_ : docstring(1, char_); }
 286         ///
 287         docstring asInput() const { return cs_.size() ? '\\' + cs_ : docstring(1, char_); }
 288
 289 private:
 290         ///
 291         docstring cs_;
 292         ///
 293         char_type char_;
 294         ///
 295         CatCode cat_;
 296 };
 297
 298
 299 ostream & operator<<(ostream & os, Token const & t)
 300 {
 301         if (t.cs().size()) {
 302                 docstring const & cs = t.cs();
 303                 // FIXME: For some strange reason, the stream operator instanciate
 304                 // a new Token before outputting the contents of t.cs().
 305                 // Because of this the line
 306                 //     os << '\\' << cs;
 307                 // below becomes recursive.
 308                 // In order to avoid that we return early:
 309                 if (cs == "\\")
 310                         return os;
 311                 os << '\\' << to_utf8(cs);
 312         }
 313         else if (t.cat() == catLetter)
 314                 os << t.character();
 315         else
 316                 os << '[' << t.character() << ',' << t.cat() << ']';
 317         return os;
 318 }
 319
 320
 321 class Parser {
 322 public:
 323         ///
 324         typedef  InsetMath::mode_type mode_type;
 325         ///
 326         typedef  Parse::flags parse_mode;
 327
 328         ///
 329         Parser(Lexer & lex, parse_mode mode);
 330         /// Only use this for reading from .lyx file format, for the reason
 331         /// see Parser::tokenize(istream &).
 332         Parser(istream & is, parse_mode mode);
 333         ///
 334         Parser(docstring const & str, parse_mode mode);
 335
 336         ///
 337         bool parse(MathAtom & at);
 338         ///
 339         bool parse(MathData & array, unsigned flags, mode_type mode);
 340         ///
 341         bool parse1(InsetMathGrid & grid, unsigned flags, mode_type mode,
 342                 bool numbered);
 343         ///
 344         MathData parse(unsigned flags, mode_type mode);
 345         ///
 346         int lineno() const { return lineno_; }
 347         ///
 348         void putback();
 349
 350 private:
 351         ///
 352         void parse2(MathAtom & at, unsigned flags, mode_type mode, bool numbered);
 353         /// get arg delimited by 'left' and 'right'
 354         docstring getArg(char_type left, char_type right);
 355         ///
 356         char_type getChar();
 357         ///
 358         void error(string const & msg);
 359         void error(docstring const & msg) { error(to_utf8(msg)); }
 360         /// dump contents to screen
 361         void dump() const;
 362         /// Only use this for reading from .lyx file format (see
 363         /// implementation for reason)
 364         void tokenize(istream & is);
 365         ///
 366         void tokenize(docstring const & s);
 367         ///
 368         void skipSpaceTokens(idocstream & is, char_type c);
 369         ///
 370         void push_back(Token const & t);
 371         ///
 372         void pop_back();
 373         ///
 374         Token const & prevToken() const;
 375         ///
 376         Token const & nextToken() const;
 377         ///
 378         Token const & getToken();
 379         /// skips spaces if any
 380         void skipSpaces();
 381         ///
 382         void lex(docstring const & s);
 383         ///
 384         bool good() const;
 385         ///
 386         docstring parse_verbatim_item();
 387         ///
 388         docstring parse_verbatim_option();
 389
 390         ///
 391         int lineno_;
 392         ///
 393         vector<Token> tokens_;
 394         ///
 395         unsigned pos_;
 396         /// Stack of active environments
 397         vector<docstring> environments_;
 398         ///
 399         parse_mode mode_;
 400         ///
 401         bool success_;
 402 };
 403
 404
 405 Parser::Parser(Lexer & lexer, parse_mode mode)
 406         : lineno_(lexer.lineNumber()), pos_(0), mode_(mode), success_(true)
 407 {
 408         tokenize(lexer.getStream());
 409         lexer.eatLine();
 410 }
 411
 412
 413 Parser::Parser(istream & is, parse_mode mode)
 414         : lineno_(0), pos_(0), mode_(mode), success_(true)
 415 {
 416         tokenize(is);
 417 }
 418
 419
 420 Parser::Parser(docstring const & str, parse_mode mode)
 421         : lineno_(0), pos_(0), mode_(mode), success_(true)
 422 {
 423         tokenize(str);
 424 }
 425
 426
 427 void Parser::push_back(Token const & t)
 428 {
 429         tokens_.push_back(t);
 430 }
 431
 432
 433 void Parser::pop_back()
 434 {
 435         tokens_.pop_back();
 436 }
 437
 438
 439 Token const & Parser::prevToken() const
 440 {
 441         static const Token dummy;
 442         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 443 }
 444
 445
 446 Token const & Parser::nextToken() const
 447 {
 448         static const Token dummy;
 449         return good() ? tokens_[pos_] : dummy;
 450 }
 451
 452
 453 Token const & Parser::getToken()
 454 {
 455         static const Token dummy;
 456         //lyxerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << endl;
 457         return good() ? tokens_[pos_++] : dummy;
 458 }
 459
 460
 461 void Parser::skipSpaces()
 462 {
 463         while (nextToken().cat() == catSpace || nextToken().cat() == catNewline)
 464                 getToken();
 465 }
 466
 467
 468 void Parser::putback()
 469 {
 470         --pos_;
 471 }
 472
 473
 474 bool Parser::good() const
 475 {
 476         return pos_ < tokens_.size();
 477 }
 478
 479
 480 char_type Parser::getChar()
 481 {
 482         if (!good()) {
 483                 error("The input stream is not well...");
 484                 putback();
 485                 return 0;
 486         }
 487         return tokens_[pos_++].character();
 488 }
 489
 490
 491 docstring Parser::getArg(char_type left, char_type right)
 492 {
 493         skipSpaces();
 494
 495         docstring result;
 496         char_type c = getChar();
 497
 498         if (c != left)
 499                 putback();
 500         else
 501                 while ((c = getChar()) != right && good())
 502                         result += c;
 503
 504         return result;
 505 }
 506
 507
 508 void Parser::skipSpaceTokens(idocstream & is, char_type c)
 509 {
 510         // skip trailing spaces
 511         while (catcode(c) == catSpace || catcode(c) == catNewline)
 512                 if (!is.get(c))
 513                         break;
 514         //lyxerr << "putting back: " << c << endl;
 515         is.putback(c);
 516 }
 517
 518
 519 void Parser::tokenize(istream & is)
 520 {
 521         // eat everything up to the next \end_inset or end of stream
 522         // and store it in s for further tokenization
 523         string s;
 524         char c;
 525         while (is.get(c)) {
 526                 s += c;
 527                 if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
 528                         s = s.substr(0, s.size() - 10);
 529                         break;
 530                 }
 531         }
 532         // Remove the space after \end_inset
 533         if (is.get(c) && c != ' ')
 534                 is.unget();
 535
 536         // tokenize buffer
 537         tokenize(from_utf8(s));
 538 }
 539
 540
 541 void Parser::tokenize(docstring const & buffer)
 542 {
 543         idocstringstream is(mode_ & Parse::VERBATIM ?
 544                 escapeSpecialChars(buffer) : buffer, ios::in | ios::binary);
 545
 546         char_type c;
 547         while (is.get(c)) {
 548                 //lyxerr << "reading c: " << c << endl;
 549
 550                 switch (catcode(c)) {
 551                         case catNewline: {
 552                                 ++lineno_;
 553                                 is.get(c);
 554                                 if (catcode(c) == catNewline)
 555                                         ; //push_back(Token("par"));
 556                                 else {
 557                                         push_back(Token('\n', catNewline));
 558                                         is.putback(c);
 559                                 }
 560                                 break;
 561                         }
 562
 563 /*
 564                         case catComment: {
 565                                 while (is.get(c) && catcode(c) != catNewline)
 566                                         ;
 567                                 ++lineno_;
 568                                 break;
 569                         }
 570 */
 571
 572                         case catEscape: {
 573                                 is.get(c);
 574                                 if (!is) {
 575                                         error("unexpected end of input");
 576                                 } else {
 577                                         docstring s(1, c);
 578                                         if (catcode(c) == catLetter) {
 579                                                 // collect letters
 580                                                 while (is.get(c) && catcode(c) == catLetter)
 581                                                         s += c;
 582                                                 skipSpaceTokens(is, c);
 583                                         }
 584                                         push_back(Token(s));
 585                                 }
 586                                 break;
 587                         }
 588
 589                         case catSuper:
 590                         case catSub: {
 591                                 push_back(Token(c, catcode(c)));
 592                                 is.get(c);
 593                                 skipSpaceTokens(is, c);
 594                                 break;
 595                         }
 596
 597                         case catIgnore: {
 598                                 if (!(mode_ & Parse::QUIET))
 599                                         lyxerr << "ignoring a char: " << int(c) << endl;
 600                                 break;
 601                         }
 602
 603                         default:
 604                                 push_back(Token(c, catcode(c)));
 605                 }
 606         }
 607
 608 #ifdef FILEDEBUG
 609         dump();
 610 #endif
 611 }
 612
 613
 614 void Parser::dump() const
 615 {
 616         lyxerr << "\nTokens: ";
 617         for (unsigned i = 0; i < tokens_.size(); ++i) {
 618                 if (i == pos_)
 619                         lyxerr << " <#> ";
 620                 lyxerr << tokens_[i];
 621         }
 622         lyxerr << " pos: " << pos_ << endl;
 623 }
 624
 625
 626 void Parser::error(string const & msg)
 627 {
 628         success_ = false;
 629         if (!(mode_ & Parse::QUIET)) {
 630                 lyxerr << "Line ~" << lineno_ << ": Math parse error: "
 631                        << msg << endl;
 632                 dump();
 633         }
 634 }
 635
 636
 637 bool Parser::parse(MathAtom & at)
 638 {
 639         skipSpaces();
 640         MathData ar;
 641         parse(ar, false, InsetMath::UNDECIDED_MODE);
 642         if (ar.size() != 1 || ar.front()->getType() == hullNone) {
 643                 if (!(mode_ & Parse::QUIET))
 644                         lyxerr << "unusual contents found: " << ar << endl;
 645                 at = MathAtom(new InsetMathPar(ar));
 646                 //if (at->nargs() > 0)
 647                 //      at.nucleus()->cell(0) = ar;
 648                 //else
 649                 //      lyxerr << "unusual contents found: " << ar << endl;
 650                 return false;
 651         }
 652         at = ar[0];
 653         return true;
 654 }
 655
 656
 657 docstring Parser::parse_verbatim_option()
 658 {
 659         skipSpaces();
 660         docstring res;
 661         if (nextToken().character() == '[') {
 662                 Token t = getToken();
 663                 for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
 664                         if (t.cat() == catBegin) {
 665                                 putback();
 666                                 res += '{' + parse_verbatim_item() + '}';
 667                         } else
 668                                 res += t.asString();
 669                 }
 670         }
 671         return res;
 672 }
 673
 674
 675 docstring Parser::parse_verbatim_item()
 676 {
 677         skipSpaces();
 678         docstring res;
 679         if (nextToken().cat() == catBegin) {
 680                 Token t = getToken();
 681                 for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
 682                         if (t.cat() == catBegin) {
 683                                 putback();
 684                                 res += '{' + parse_verbatim_item() + '}';
 685                         }
 686                         else
 687                                 res += t.asString();
 688                 }
 689         }
 690         return res;
 691 }
 692
 693
 694 MathData Parser::parse(unsigned flags, mode_type mode)
 695 {
 696         MathData ar;
 697         parse(ar, flags, mode);
 698         return ar;
 699 }
 700
 701
 702 bool Parser::parse(MathData & array, unsigned flags, mode_type mode)
 703 {
 704         InsetMathGrid grid(1, 1);
 705         parse1(grid, flags, mode, false);
 706         array = grid.cell(0);
 707         return success_;
 708 }
 709
 710
 711 void Parser::parse2(MathAtom & at, const unsigned flags, const mode_type mode,
 712         const bool numbered)
 713 {
 714         parse1(*(at.nucleus()->asGridInset()), flags, mode, numbered);
 715 }
 716
 717
 718 bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
 719         const mode_type mode, const bool numbered)
 720 {
 721         int limits = 0;
 722         InsetMathGrid::row_type cellrow = 0;
 723         InsetMathGrid::col_type cellcol = 0;
 724         MathData * cell = &grid.cell(grid.index(cellrow, cellcol));
 725
 726         if (grid.asHullInset())
 727                 grid.asHullInset()->numbered(cellrow, numbered);
 728
 729         //dump();
 730         //lyxerr << " flags: " << flags << endl;
 731         //lyxerr << " mode: " << mode  << endl;
 732         //lyxerr << "grid: " << grid << endl;
 733
 734         while (good()) {
 735                 Token const & t = getToken();
 736
 737 #ifdef FILEDEBUG
 738                 lyxerr << "t: " << t << " flags: " << flags << endl;
 739                 lyxerr << "mode: " << mode  << endl;
 740                 cell->dump();
 741                 lyxerr << endl;
 742 #endif
 743
 744                 if (flags & FLAG_ITEM) {
 745
 746                         if (t.cat() == catBegin) {
 747                                 // skip the brace and collect everything to the next matching
 748                                 // closing brace
 749                                 parse1(grid, FLAG_BRACE_LAST, mode, numbered);
 750                                 return success_;
 751                         }
 752
 753                         // handle only this single token, leave the loop if done
 754                         flags = FLAG_LEAVE;
 755                 }
 756
 757
 758                 if (flags & FLAG_BRACED) {
 759                         if (t.cat() == catSpace)
 760                                 continue;
 761
 762                         if (t.cat() != catBegin) {
 763                                 error("opening brace expected");
 764                                 return success_;
 765                         }
 766
 767                         // skip the brace and collect everything to the next matching
 768                         // closing brace
 769                         flags = FLAG_BRACE_LAST;
 770                 }
 771
 772
 773                 if (flags & FLAG_OPTION) {
 774                         if (t.cat() == catOther && t.character() == '[') {
 775                                 MathData ar;
 776                                 parse(ar, FLAG_BRACK_LAST, mode);
 777                                 cell->append(ar);
 778                         } else {
 779                                 // no option found, put back token and we are done
 780                                 putback();
 781                         }
 782                         return success_;
 783                 }
 784
 785                 //
 786                 // cat codes
 787                 //
 788                 if (t.cat() == catMath) {
 789                         if (mode != InsetMath::MATH_MODE) {
 790                                 // we are inside some text mode thingy, so opening new math is allowed
 791                                 Token const & n = getToken();
 792                                 if (n.cat() == catMath) {
 793                                         // TeX's $$...$$ syntax for displayed math
 794                                         cell->push_back(MathAtom(new InsetMathHull(hullEquation)));
 795                                         parse2(cell->back(), FLAG_SIMPLE, InsetMath::MATH_MODE, false);
 796                                         getToken(); // skip the second '$' token
 797                                 } else {
 798                                         // simple $...$  stuff
 799                                         putback();
 800                                         cell->push_back(MathAtom(new InsetMathHull(hullSimple)));
 801                                         parse2(cell->back(), FLAG_SIMPLE, InsetMath::MATH_MODE, false);
 802                                 }
 803                         }
 804
 805                         else if (flags & FLAG_SIMPLE) {
 806                                 // this is the end of the formula
 807                                 return success_;
 808                         }
 809
 810                         else {
 811                                 error("something strange in the parser");
 812                                 break;
 813                         }
 814                 }
 815
 816                 else if (t.cat() == catLetter)
 817                         cell->push_back(MathAtom(new InsetMathChar(t.character())));
 818
 819                 else if (t.cat() == catSpace && mode != InsetMath::MATH_MODE) {
 820                         if (cell->empty() || cell->back()->getChar() != ' ')
 821                                 cell->push_back(MathAtom(new InsetMathChar(t.character())));
 822                 }
 823
 824                 else if (t.cat() == catNewline && mode != InsetMath::MATH_MODE) {
 825                         if (cell->empty() || cell->back()->getChar() != ' ')
 826                                 cell->push_back(MathAtom(new InsetMathChar(' ')));
 827                 }
 828
 829                 else if (t.cat() == catParameter) {
 830                         Token const & n = getToken();
 831                         cell->push_back(MathAtom(new MathMacroArgument(n.character()-'0')));
 832                 }
 833
 834                 else if (t.cat() == catActive)
 835                         cell->push_back(MathAtom(new InsetMathChar(t.character())));
 836
 837                 else if (t.cat() == catBegin) {
 838                         MathData ar;
 839                         parse(ar, FLAG_BRACE_LAST, mode);
 840                         // do not create a BraceInset if they were written by LyX
 841                         // this helps to keep the annoyance of  "a choose b"  to a minimum
 842                         if (ar.size() == 1 && ar[0]->extraBraces())
 843                                 cell->append(ar);
 844                         else
 845                                 cell->push_back(MathAtom(new InsetMathBrace(ar)));
 846                 }
 847
 848                 else if (t.cat() == catEnd) {
 849                         if (flags & FLAG_BRACE_LAST)
 850                                 return success_;
 851                         error("found '}' unexpectedly");
 852                         //LASSERT(false, /**/);
 853                         //add(cell, '}', LM_TC_TEX);
 854                 }
 855
 856                 else if (t.cat() == catAlign) {
 857                         //lyxerr << " column now " << (cellcol + 1)
 858                         //       << " max: " << grid.ncols() << endl;
 859                         if (flags & FLAG_ALIGN)
 860                                 return success_;
 861                         if (addCol(grid, cellcol))
 862                                 cell = &grid.cell(grid.index(cellrow, cellcol));
 863                 }
 864
 865                 else if (t.cat() == catSuper || t.cat() == catSub) {
 866                         bool up = (t.cat() == catSuper);
 867                         // we need no new script inset if the last thing was a scriptinset,
 868                         // which has that script already not the same script already
 869                         if (!cell->size())
 870                                 cell->push_back(MathAtom(new InsetMathScript(up)));
 871                         else if (cell->back()->asScriptInset() &&
 872                                         !cell->back()->asScriptInset()->has(up))
 873                                 cell->back().nucleus()->asScriptInset()->ensure(up);
 874                         else if (cell->back()->asScriptInset())
 875                                 cell->push_back(MathAtom(new InsetMathScript(up)));
 876                         else
 877                                 cell->back() = MathAtom(new InsetMathScript(cell->back(), up));
 878                         InsetMathScript * p = cell->back().nucleus()->asScriptInset();
 879                         // special handling of {}-bases
 880                         // Here we could remove the brace inset for things
 881                         // like {a'}^2 and add the braces back in
 882                         // InsetMathScript::write().
 883                         // We do not do it, since it is not possible to detect
 884                         // reliably whether the braces are needed because the
 885                         // nucleus contains more than one symbol, or whether
 886                         // they are needed for unknown commands like \xx{a}_0
 887                         // or \yy{a}{b}_0. This was done in revision 14819
 888                         // in an unreliable way. See this thread
 889                         // http://www.mail-archive.com/lyx-devel%40lists.lyx.org/msg104917.html
 890                         // for more details.
 891                         parse(p->cell(p->idxOfScript(up)), FLAG_ITEM, mode);
 892                         if (limits) {
 893                                 p->limits(limits);
 894                                 limits = 0;
 895                         }
 896                 }
 897
 898                 else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
 899                         //lyxerr << "finished reading option" << endl;
 900                         return success_;
 901                 }
 902
 903                 else if (t.cat() == catOther)
 904                         cell->push_back(MathAtom(new InsetMathChar(t.character())));
 905
 906                 else if (t.cat() == catComment) {
 907                         docstring s;
 908                         while (good()) {
 909                                 Token const & t = getToken();
 910                                 if (t.cat() == catNewline)
 911                                         break;
 912                                 s += t.asString();
 913                         }
 914                         cell->push_back(MathAtom(new InsetMathComment(s)));
 915                         skipSpaces();
 916                 }
 917
 918                 //
 919                 // control sequences
 920                 //
 921
 922                 else if (t.cs() == "lyxlock") {
 923                         if (cell->size())
 924                                 cell->back().nucleus()->lock(true);
 925                 }
 926
 927                 else if ((t.cs() == "global" && nextToken().cs() == "def") ||
 928                          t.cs() == "def") {
 929                         if (t.cs() == "global")
 930                                 getToken();
 931
 932                         // get name
 933                         docstring name = getToken().cs();
 934
 935                         // read parameters
 936                         int nargs = 0;
 937                         docstring pars;
 938                         while (good() && nextToken().cat() != catBegin) {
 939                                 pars += getToken().cs();
 940                                 ++nargs;
 941                         }
 942                         nargs /= 2;
 943
 944                         // read definition
 945                         MathData def;
 946                         parse(def, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
 947
 948                         // is a version for display attached?
 949                         skipSpaces();
 950                         MathData display;
 951                         if (nextToken().cat() == catBegin)
 952                                 parse(display, FLAG_ITEM, InsetMath::MATH_MODE);
 953
 954                         cell->push_back(MathAtom(new MathMacroTemplate(name, nargs,
 955                                0, MacroTypeDef, vector<MathData>(), def, display)));
 956                 }
 957
 958                 else if (t.cs() == "newcommand" ||
 959                          t.cs() == "renewcommand" ||
 960                          t.cs() == "newlyxcommand") {
 961                         // get name
 962                         if (getToken().cat() != catBegin) {
 963                                 error("'{' in \\newcommand expected (1) ");
 964                                 return success_;
 965                         }
 966                         docstring name = getToken().cs();
 967                         if (getToken().cat() != catEnd) {
 968                                 error("'}' in \\newcommand expected");
 969                                 return success_;
 970                         }
 971
 972                         // get arity
 973                         docstring const arg = getArg('[', ']');
 974                         int nargs = 0;
 975                         if (!arg.empty())
 976                                 nargs = convert<int>(arg);
 977
 978                         // optional argument given?
 979                         skipSpaces();
 980                         int optionals = 0;
 981                         vector<MathData> optionalValues;
 982                         while (nextToken().character() == '[') {
 983                                 getToken();
 984                                 optionalValues.push_back(MathData());
 985                                 parse(optionalValues[optionals], FLAG_BRACK_LAST, mode);
 986                                 ++optionals;
 987                         }
 988
 989                         MathData def;
 990                         parse(def, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
 991
 992                         // is a version for display attached?
 993                         skipSpaces();
 994                         MathData display;
 995                         if (nextToken().cat() == catBegin)
 996                                 parse(display, FLAG_ITEM, InsetMath::MATH_MODE);
 997
 998                         cell->push_back(MathAtom(new MathMacroTemplate(name, nargs,
 999                                 optionals, MacroTypeNewcommand, optionalValues, def, display)));
1000
1001                 }
1002
1003                 else if (t.cs() == "newcommandx" ||
1004                          t.cs() == "renewcommandx") {
1005                         // \newcommandx{\foo}[2][usedefault, addprefix=\global,1=default]{#1,#2}
1006                         // get name
1007                         docstring name;
1008                         if (nextToken().cat() == catBegin) {
1009                                 getToken();
1010                                 name = getToken().cs();
1011                                 if (getToken().cat() != catEnd) {
1012                                         error("'}' in \\newcommandx expected");
1013                                         return success_;
1014                                 }
1015                         } else
1016                                 name = getToken().cs();
1017
1018                         // get arity
1019                         docstring const arg = getArg('[', ']');
1020                         if (arg.empty()) {
1021                                 error("[num] in \\newcommandx expected");
1022                                 return success_;
1023                         }
1024                         int nargs = convert<int>(arg);
1025
1026                         // get options
1027                         int optionals = 0;
1028                         vector<MathData> optionalValues;
1029                         if (nextToken().character() == '[') {
1030                                 // skip '['
1031                                 getToken();
1032
1033                                 // handle 'opt=value' options, separated by ','.
1034                                 skipSpaces();
1035                                 while (nextToken().character() != ']' && good()) {
1036                                         if (nextToken().character() >= '1'
1037                                             && nextToken().character() <= '9') {
1038                                                 // optional value -> get parameter number
1039                                                 int n = getChar() - '0';
1040                                                 if (n > nargs) {
1041                                                         error("Arity of \\newcommandx too low "
1042                                                               "for given optional parameter.");
1043                                                         return success_;
1044                                                 }
1045
1046                                                 // skip '='
1047                                                 if (getToken().character() != '=') {
1048                                                         error("'=' and optional parameter value "
1049                                                               "expected for \\newcommandx");
1050                                                         return success_;
1051                                                 }
1052
1053                                                 // get value
1054                                                 int optNum = max(size_t(n), optionalValues.size());
1055                                                 optionalValues.resize(optNum);
1056                                                 optionalValues[n - 1].clear();
1057                                                 while (nextToken().character() != ']'
1058                                                        && nextToken().character() != ',') {
1059                                                         MathData data;
1060                                                         parse(data, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
1061                                                         optionalValues[n - 1].append(data);
1062                                                 }
1063                                                 optionals = max(n, optionals);
1064                                         } else if (nextToken().cat() == catLetter) {
1065                                                 // we in fact ignore every non-optional
1066                                                 // parameter
1067
1068                                                 // get option name
1069                                                 docstring opt;
1070                                                 while (nextToken().cat() == catLetter)
1071                                                         opt += getChar();
1072
1073                                                 // value?
1074                                                 skipSpaces();
1075                                                 MathData value;
1076                                                 if (nextToken().character() == '=') {
1077                                                         getToken();
1078                                                         while (nextToken().character() != ']'
1079                                                                 && nextToken().character() != ',')
1080                                                                 parse(value, FLAG_ITEM,
1081                                                                       InsetMath::UNDECIDED_MODE);
1082                                                 }
1083                                         } else {
1084                                                 error("option for \\newcommandx expected");
1085                                                 return success_;
1086                                         }
1087
1088                                         // skip komma
1089                                         skipSpaces();
1090                                         if (nextToken().character() == ',') {
1091                                                 getChar();
1092                                                 skipSpaces();
1093                                         } else if (nextToken().character() != ']') {
1094                                                 error("Expecting ',' or ']' in options "
1095                                                       "of \\newcommandx");
1096                                                 return success_;
1097                                         }
1098                                 }
1099
1100                                 // skip ']'
1101                                 if (!good())
1102                                         return success_;
1103                                 getToken();
1104                         }
1105
1106                         // get definition
1107                         MathData def;
1108                         parse(def, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
1109
1110                         // is a version for display attached?
1111                         skipSpaces();
1112                         MathData display;
1113                         if (nextToken().cat() == catBegin)
1114                                 parse(display, FLAG_ITEM, InsetMath::MATH_MODE);
1115
1116                         cell->push_back(MathAtom(new MathMacroTemplate(name, nargs,
1117                                 optionals, MacroTypeNewcommandx, optionalValues, def,
1118                                 display)));
1119                 }
1120
1121                 else if (t.cs() == "(") {
1122                         cell->push_back(MathAtom(new InsetMathHull(hullSimple)));
1123                         parse2(cell->back(), FLAG_SIMPLE2, InsetMath::MATH_MODE, false);
1124                 }
1125
1126                 else if (t.cs() == "[") {
1127                         cell->push_back(MathAtom(new InsetMathHull(hullEquation)));
1128                         parse2(cell->back(), FLAG_EQUATION, InsetMath::MATH_MODE, false);
1129                 }
1130
1131                 else if (t.cs() == "protect")
1132                         // ignore \\protect, will hopefully be re-added during output
1133                         ;
1134
1135                 else if (t.cs() == "end") {
1136                         if (flags & FLAG_END) {
1137                                 // eat environment name
1138                                 docstring const name = getArg('{', '}');
1139                                 if (environments_.empty())
1140                                         error("'found \\end{" + name +
1141                                               "}' without matching '\\begin{" +
1142                                               name + "}'");
1143                                 else if (name != environments_.back())
1144                                         error("'\\end{" + name +
1145                                               "}' does not match '\\begin{" +
1146                                               environments_.back() + "}'");
1147                                 else {
1148                                         environments_.pop_back();
1149                                         // Delete empty last row in matrix
1150                                         // like insets.
1151                                         // If you abuse InsetMathGrid for
1152                                         // non-matrix like structures you
1153                                         // probably need to refine this test.
1154                                         // Right now we only have to test for
1155                                         // single line hull insets.
1156                                         if (grid.nrows() > 1)
1157                                                 delEmptyLastRow(grid);
1158                                         return success_;
1159                                 }
1160                         } else
1161                                 error("found 'end' unexpectedly");
1162                 }
1163
1164                 else if (t.cs() == ")") {
1165                         if (flags & FLAG_SIMPLE2)
1166                                 return success_;
1167                         error("found '\\)' unexpectedly");
1168                 }
1169
1170                 else if (t.cs() == "]") {
1171                         if (flags & FLAG_EQUATION)
1172                                 return success_;
1173                         error("found '\\]' unexpectedly");
1174                 }
1175
1176                 else if (t.cs() == "\\") {
1177                         if (flags & FLAG_ALIGN)
1178                                 return success_;
1179                         bool added = false;
1180                         if (nextToken().asInput() == "*") {
1181                                 getToken();
1182                                 added = addRow(grid, cellrow, docstring(), false);
1183                         } else if (good())
1184                                 added = addRow(grid, cellrow, getArg('[', ']'));
1185                         else
1186                                 error("missing token after \\\\");
1187                         if (added) {
1188                                 cellcol = 0;
1189                                 if (grid.asHullInset())
1190                                         grid.asHullInset()->numbered(
1191                                                         cellrow, numbered);
1192                                 cell = &grid.cell(grid.index(cellrow,
1193                                                              cellcol));
1194                         }
1195                 }
1196
1197 #if 0
1198                 else if (t.cs() == "multicolumn") {
1199                         // extract column count and insert dummy cells
1200                         MathData count;
1201                         parse(count, FLAG_ITEM, mode);
1202                         int cols = 1;
1203                         if (!extractNumber(count, cols)) {
1204                                 success_ = false;
1205                                 lyxerr << " can't extract number of cells from " << count << endl;
1206                         }
1207                         // resize the table if necessary
1208                         for (int i = 0; i < cols; ++i) {
1209                                 if (addCol(grid, cellcol)) {
1210                                         cell = &grid.cell(grid.index(
1211                                                         cellrow, cellcol));
1212                                         // mark this as dummy
1213                                         grid.cellinfo(grid.index(
1214                                                 cellrow, cellcol)).dummy_ = true;
1215                                 }
1216                         }
1217                         // the last cell is the real thing, not a dummy
1218                         grid.cellinfo(grid.index(cellrow, cellcol)).dummy_ = false;
1219
1220                         // read special alignment
1221                         MathData align;
1222                         parse(align, FLAG_ITEM, mode);
1223                         //grid.cellinfo(grid.index(cellrow, cellcol)).align_ = extractString(align);
1224
1225                         // parse the remaining contents into the "real" cell
1226                         parse(*cell, FLAG_ITEM, mode);
1227                 }
1228 #endif
1229
1230                 else if (t.cs() == "limits")
1231                         limits = 1;
1232
1233                 else if (t.cs() == "nolimits")
1234                         limits = -1;
1235
1236                 else if (t.cs() == "nonumber") {
1237                         if (grid.asHullInset())
1238                                 grid.asHullInset()->numbered(cellrow, false);
1239                 }
1240
1241                 else if (t.cs() == "number") {
1242                         if (grid.asHullInset())
1243                                 grid.asHullInset()->numbered(cellrow, true);
1244                 }
1245
1246                 else if (t.cs() == "hline") {
1247                         grid.rowinfo(cellrow).lines_ ++;
1248                 }
1249
1250                 else if (t.cs() == "sqrt") {
1251                         MathData ar;
1252                         parse(ar, FLAG_OPTION, mode);
1253                         if (ar.size()) {
1254                                 cell->push_back(MathAtom(new InsetMathRoot));
1255                                 cell->back().nucleus()->cell(0) = ar;
1256                                 parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
1257                         } else {
1258                                 cell->push_back(MathAtom(new InsetMathSqrt));
1259                                 parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1260                         }
1261                 }
1262
1263                 else if (t.cs() == "unit") {
1264                         // Allowed formats \unit[val]{unit}
1265                         MathData ar;
1266                         parse(ar, FLAG_OPTION, mode);
1267                         if (ar.size()) {
1268                                 cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::UNIT)));
1269                                 cell->back().nucleus()->cell(0) = ar;
1270                                 parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
1271                         } else {
1272                                 cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::UNIT, 1)));
1273                                 parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1274                         }
1275                 }
1276                 else if (t.cs() == "unitfrac") {
1277                         // Here allowed formats are \unitfrac[val]{num}{denom}
1278                         MathData ar;
1279                         parse(ar, FLAG_OPTION, mode);
1280                         if (ar.size()) {
1281                                 cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::UNITFRAC, 3)));
1282                                 cell->back().nucleus()->cell(2) = ar;
1283                         } else {
1284                                 cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::UNITFRAC)));
1285                         }
1286                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1287                         parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
1288                 }
1289
1290                 else if (t.cs() == "xrightarrow" || t.cs() == "xleftarrow") {
1291                         cell->push_back(createInsetMath(t.cs()));
1292                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, mode);
1293                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1294                 }
1295
1296                 else if (t.cs() == "ref" || t.cs() == "eqref" || t.cs() == "prettyref"
1297                           || t.cs() == "pageref" || t.cs() == "vpageref" || t.cs() == "vref") {
1298                         cell->push_back(MathAtom(new InsetMathRef(t.cs())));
1299                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, mode);
1300                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1301                 }
1302
1303                 else if (t.cs() == "left") {
1304                         skipSpaces();
1305                         Token const & tl = getToken();
1306                         // \| and \Vert are equivalent, and InsetMathDelim
1307                         // can't handle \|
1308                         // FIXME: fix this in InsetMathDelim itself!
1309                         docstring const l = tl.cs() == "|" ? from_ascii("Vert") : tl.asString();
1310                         MathData ar;
1311                         parse(ar, FLAG_RIGHT, mode);
1312                         if (!good())
1313                                 break;
1314                         skipSpaces();
1315                         Token const & tr = getToken();
1316                         docstring const r = tr.cs() == "|" ? from_ascii("Vert") : tr.asString();
1317                         cell->push_back(MathAtom(new InsetMathDelim(l, r, ar)));
1318                 }
1319
1320                 else if (t.cs() == "right") {
1321                         if (flags & FLAG_RIGHT)
1322                                 return success_;
1323                         //lyxerr << "got so far: '" << cell << "'" << endl;
1324                         error("Unmatched right delimiter");
1325                         return success_;
1326                 }
1327
1328                 else if (t.cs() == "begin") {
1329                         docstring const name = getArg('{', '}');
1330                         environments_.push_back(name);
1331
1332                         if (name == "array" || name == "subarray") {
1333                                 docstring const valign = parse_verbatim_option() + 'c';
1334                                 docstring const halign = parse_verbatim_item();
1335                                 cell->push_back(MathAtom(new InsetMathArray(name,
1336                                         InsetMathGrid::guessColumns(halign), 1, (char)valign[0], halign)));
1337                                 parse2(cell->back(), FLAG_END, mode, false);
1338                         }
1339
1340                         else if (name == "tabular") {
1341                                 docstring const valign = parse_verbatim_option() + 'c';
1342                                 docstring const halign = parse_verbatim_item();
1343                                 cell->push_back(MathAtom(new InsetMathTabular(name,
1344                                         InsetMathGrid::guessColumns(halign), 1, (char)valign[0], halign)));
1345                                 parse2(cell->back(), FLAG_END, InsetMath::TEXT_MODE, false);
1346                         }
1347
1348                         else if (name == "split" || name == "cases") {
1349                                 cell->push_back(createInsetMath(name));
1350                                 parse2(cell->back(), FLAG_END, mode, false);
1351                         }
1352
1353                         else if (name == "alignedat") {
1354                                 docstring const valign = parse_verbatim_option() + 'c';
1355                                 // ignore this for a while
1356                                 getArg('{', '}');
1357                                 cell->push_back(MathAtom(new InsetMathSplit(name, (char)valign[0])));
1358                                 parse2(cell->back(), FLAG_END, mode, false);
1359                         }
1360
1361                         else if (name == "math") {
1362                                 cell->push_back(MathAtom(new InsetMathHull(hullSimple)));
1363                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, true);
1364                         }
1365
1366                         else if (name == "equation" || name == "equation*"
1367                                         || name == "displaymath") {
1368                                 cell->push_back(MathAtom(new InsetMathHull(hullEquation)));
1369                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, (name == "equation"));
1370                         }
1371
1372                         else if (name == "eqnarray" || name == "eqnarray*") {
1373                                 cell->push_back(MathAtom(new InsetMathHull(hullEqnArray)));
1374                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1375                         }
1376
1377                         else if (name == "align" || name == "align*") {
1378                                 cell->push_back(MathAtom(new InsetMathHull(hullAlign)));
1379                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1380                         }
1381
1382                         else if (name == "flalign" || name == "flalign*") {
1383                                 cell->push_back(MathAtom(new InsetMathHull(hullFlAlign)));
1384                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1385                         }
1386
1387                         else if (name == "alignat" || name == "alignat*") {
1388                                 // ignore this for a while
1389                                 getArg('{', '}');
1390                                 cell->push_back(MathAtom(new InsetMathHull(hullAlignAt)));
1391                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1392                         }
1393
1394                         else if (name == "xalignat" || name == "xalignat*") {
1395                                 // ignore this for a while
1396                                 getArg('{', '}');
1397                                 cell->push_back(MathAtom(new InsetMathHull(hullXAlignAt)));
1398                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1399                         }
1400
1401                         else if (name == "xxalignat") {
1402                                 // ignore this for a while
1403                                 getArg('{', '}');
1404                                 cell->push_back(MathAtom(new InsetMathHull(hullXXAlignAt)));
1405                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1406                         }
1407
1408                         else if (name == "multline" || name == "multline*") {
1409                                 cell->push_back(MathAtom(new InsetMathHull(hullMultline)));
1410                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1411                         }
1412
1413                         else if (name == "gather" || name == "gather*") {
1414                                 cell->push_back(MathAtom(new InsetMathHull(hullGather)));
1415                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1416                         }
1417
1418                         else if (latexkeys const * l = in_word_set(name)) {
1419                                 if (l->inset == "matrix") {
1420                                         cell->push_back(createInsetMath(name));
1421                                         parse2(cell->back(), FLAG_END, mode, false);
1422                                 } else if (l->inset == "split") {
1423                                         docstring const valign = parse_verbatim_option() + 'c';
1424                                         cell->push_back(MathAtom(new InsetMathSplit(name, (char)valign[0])));
1425                                         parse2(cell->back(), FLAG_END, mode, false);
1426                                 } else {
1427                                         success_ = false;
1428                                         if (!(mode_ & Parse::QUIET)) {
1429                                                 dump();
1430                                                 lyxerr << "found math environment `"
1431                                                        << to_utf8(name)
1432                                                        << "' in symbols file with unsupported inset `"
1433                                                        << to_utf8(l->inset)
1434                                                        << "'." << endl;
1435                                         }
1436                                         // create generic environment inset
1437                                         cell->push_back(MathAtom(new InsetMathEnv(name)));
1438                                         parse(cell->back().nucleus()->cell(0), FLAG_END, mode);
1439                                 }
1440                         }
1441
1442                         else {
1443                                 success_ = false;
1444                                 if (!(mode_ & Parse::QUIET)) {
1445                                         dump();
1446                                         lyxerr << "found unknown math environment '"
1447                                                << to_utf8(name) << "'" << endl;
1448                                 }
1449                                 // create generic environment inset
1450                                 cell->push_back(MathAtom(new InsetMathEnv(name)));
1451                                 parse(cell->back().nucleus()->cell(0), FLAG_END, mode);
1452                         }
1453                 }
1454
1455                 else if (t.cs() == "kern") {
1456                         // FIXME: A hack...
1457                         docstring s;
1458                         while (true) {
1459                                 Token const & t = getToken();
1460                                 if (!good()) {
1461                                         putback();
1462                                         break;
1463                                 }
1464                                 s += t.character();
1465                                 if (isValidLength(to_utf8(s)))
1466                                         break;
1467                         }
1468                         cell->push_back(MathAtom(new InsetMathKern(s)));
1469                 }
1470
1471                 else if (t.cs() == "label") {
1472                         // FIXME: This is swallowed in inline formulas
1473                         docstring label = parse_verbatim_item();
1474                         MathData ar;
1475                         asArray(label, ar);
1476                         if (grid.asHullInset()) {
1477                                 grid.asHullInset()->label(cellrow, label);
1478                         } else {
1479                                 cell->push_back(createInsetMath(t.cs()));
1480                                 cell->push_back(MathAtom(new InsetMathBrace(ar)));
1481                         }
1482                 }
1483
1484                 else if (t.cs() == "choose" || t.cs() == "over"
1485                                 || t.cs() == "atop" || t.cs() == "brace"
1486                                 || t.cs() == "brack") {
1487                         MathAtom at = createInsetMath(t.cs());
1488                         at.nucleus()->cell(0) = *cell;
1489                         cell->clear();
1490                         parse(at.nucleus()->cell(1), flags, mode);
1491                         cell->push_back(at);
1492                         return success_;
1493                 }
1494
1495                 else if (t.cs() == "color") {
1496                         docstring const color = parse_verbatim_item();
1497                         cell->push_back(MathAtom(new InsetMathColor(true, color)));
1498                         parse(cell->back().nucleus()->cell(0), flags, mode);
1499                         return success_;
1500                 }
1501
1502                 else if (t.cs() == "textcolor") {
1503                         docstring const color = parse_verbatim_item();
1504                         cell->push_back(MathAtom(new InsetMathColor(false, color)));
1505                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, InsetMath::TEXT_MODE);
1506                 }
1507
1508                 else if (t.cs() == "normalcolor") {
1509                         cell->push_back(createInsetMath(t.cs()));
1510                         parse(cell->back().nucleus()->cell(0), flags, mode);
1511                         return success_;
1512                 }
1513
1514                 else if (t.cs() == "substack") {
1515                         cell->push_back(createInsetMath(t.cs()));
1516                         parse2(cell->back(), FLAG_ITEM, mode, false);
1517                 }
1518
1519                 else if (t.cs() == "xymatrix") {
1520                         odocstringstream os;
1521                         while (good() && nextToken().cat() != catBegin)
1522                                 os << getToken().asInput();
1523                         cell->push_back(createInsetMath(t.cs() + os.str()));
1524                         parse2(cell->back(), FLAG_ITEM, mode, false);
1525                 }
1526
1527                 else if (t.cs() == "framebox" || t.cs() == "makebox") {
1528                         cell->push_back(createInsetMath(t.cs()));
1529                         parse(cell->back().nucleus()->cell(0), FLAG_OPTION, InsetMath::TEXT_MODE);
1530                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, InsetMath::TEXT_MODE);
1531                         parse(cell->back().nucleus()->cell(2), FLAG_ITEM, InsetMath::TEXT_MODE);
1532                 }
1533
1534                 else if (t.cs() == "tag") {
1535                         if (nextToken().character() == '*') {
1536                                 getToken();
1537                                 cell->push_back(createInsetMath(t.cs() + '*'));
1538                         } else
1539                                 cell->push_back(createInsetMath(t.cs()));
1540                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, InsetMath::TEXT_MODE);
1541                 }
1542
1543 #if 0
1544                 else if (t.cs() == "infer") {
1545                         MathData ar;
1546                         parse(ar, FLAG_OPTION, mode);
1547                         cell->push_back(createInsetMath(t.cs()));
1548                         parse2(cell->back(), FLAG_ITEM, mode, false);
1549                 }
1550
1551                 // Disabled
1552                 else if (1 && t.cs() == "ar") {
1553                         auto_ptr<InsetMathXYArrow> p(new InsetMathXYArrow);
1554                         // try to read target
1555                         parse(p->cell(0), FLAG_OTPTION, mode);
1556                         // try to read label
1557                         if (nextToken().cat() == catSuper || nextToken().cat() == catSub) {
1558                                 p->up_ = nextToken().cat() == catSuper;
1559                                 getToken();
1560                                 parse(p->cell(1), FLAG_ITEM, mode);
1561                                 //lyxerr << "read label: " << p->cell(1) << endl;
1562                         }
1563
1564                         cell->push_back(MathAtom(p.release()));
1565                         //lyxerr << "read cell: " << cell << endl;
1566                 }
1567 #endif
1568
1569                 else if (t.cs() == "lyxmathsym") {
1570                         skipSpaces();
1571                         if (getToken().cat() != catBegin) {
1572                                 error("'{' expected in \\" + t.cs());
1573                                 return success_;
1574                         }
1575                         int count = 0;
1576                         docstring cmd;
1577                         CatCode cat = nextToken().cat();
1578                         while (good() && (count || cat != catEnd)) {
1579                                 if (cat == catBegin)
1580                                         ++count;
1581                                 else if (cat == catEnd)
1582                                         --count;
1583                                 cmd += getToken().asInput();
1584                                 cat = nextToken().cat();
1585                         }
1586                         if (getToken().cat() != catEnd) {
1587                                 error("'}' expected in \\" + t.cs());
1588                                 return success_;
1589                         }
1590                         docstring rem;
1591                         cmd = Encodings::fromLaTeXCommand(cmd, rem);
1592                         for (size_t i = 0; i < cmd.size(); ++i)
1593                                 cell->push_back(MathAtom(new InsetMathChar(cmd[i])));
1594                         if (rem.size()) {
1595                                 MathAtom at = createInsetMath(t.cs());
1596                                 cell->push_back(at);
1597                                 MathData ar;
1598                                 if (!mathed_parse_cell(ar, '{' + rem + '}', mode_))
1599                                         success_ = false;;
1600                                 cell->append(ar);
1601                         }
1602                 }
1603
1604                 else if (t.cs().size()) {
1605                         latexkeys const * l = in_word_set(t.cs());
1606                         if (l) {
1607                                 if (l->inset == "big") {
1608                                         skipSpaces();
1609                                         docstring const delim = getToken().asInput();
1610                                         if (InsetMathBig::isBigInsetDelim(delim))
1611                                                 cell->push_back(MathAtom(
1612                                                         new InsetMathBig(t.cs(), delim)));
1613                                         else {
1614                                                 cell->push_back(createInsetMath(t.cs()));
1615                                                 putback();
1616                                         }
1617                                 }
1618
1619                                 else if (l->inset == "font") {
1620                                         cell->push_back(createInsetMath(t.cs()));
1621                                         parse(cell->back().nucleus()->cell(0),
1622                                                 FLAG_ITEM, asMode(mode, l->extra));
1623                                 }
1624
1625                                 else if (l->inset == "oldfont") {
1626                                         cell->push_back(createInsetMath(t.cs()));
1627                                         parse(cell->back().nucleus()->cell(0),
1628                                                 flags | FLAG_ALIGN, asMode(mode, l->extra));
1629                                         if (prevToken().cat() != catAlign &&
1630                                             prevToken().cs() != "\\")
1631                                                 return success_;
1632                                         putback();
1633                                 }
1634
1635                                 else if (l->inset == "style") {
1636                                         cell->push_back(createInsetMath(t.cs()));
1637                                         parse(cell->back().nucleus()->cell(0),
1638                                                 flags | FLAG_ALIGN, mode);
1639                                         if (prevToken().cat() != catAlign &&
1640                                             prevToken().cs() != "\\")
1641                                                 return success_;
1642                                         putback();
1643                                 }
1644
1645                                 else {
1646                                         MathAtom at = createInsetMath(t.cs());
1647                                         for (InsetMath::idx_type i = 0; i < at->nargs(); ++i)
1648                                                 parse(at.nucleus()->cell(i),
1649                                                         FLAG_ITEM, asMode(mode, l->extra));
1650                                         cell->push_back(at);
1651                                 }
1652                         }
1653
1654                         else {
1655                                 bool is_unicode_symbol = false;
1656                                 if (mode == InsetMath::TEXT_MODE) {
1657                                         int num_tokens = 0;
1658                                         docstring cmd = prevToken().asInput();
1659                                         skipSpaces();
1660                                         CatCode cat = nextToken().cat();
1661                                         if (cat == catBegin) {
1662                                                 int count = 0;
1663                                                 while (good() && (count || cat != catEnd)) {
1664                                                         cat = nextToken().cat();
1665                                                         cmd += getToken().asInput();
1666                                                         ++num_tokens;
1667                                                         if (cat == catBegin)
1668                                                                 ++count;
1669                                                         else if (cat == catEnd)
1670                                                                 --count;
1671                                                 }
1672                                         }
1673                                         bool is_combining;
1674                                         char_type c =
1675                                                 Encodings::fromLaTeXCommand(cmd, is_combining);
1676                                         if (is_combining) {
1677                                                 if (cat == catLetter)
1678                                                         cmd += '{';
1679                                                 cmd += getToken().asInput();
1680                                                 ++num_tokens;
1681                                                 if (cat == catLetter)
1682                                                         cmd += '}';
1683                                                 c = Encodings::fromLaTeXCommand(cmd, is_combining);
1684                                         }
1685                                         if (c) {
1686                                                 is_unicode_symbol = true;
1687                                                 cell->push_back(MathAtom(new InsetMathChar(c)));
1688                                         } else {
1689                                                 while (num_tokens--)
1690                                                         putback();
1691                                         }
1692                                 }
1693                                 if (!is_unicode_symbol) {
1694                                         MathAtom at = createInsetMath(t.cs());
1695                                         InsetMath::mode_type m = mode;
1696                                         //if (m == InsetMath::UNDECIDED_MODE)
1697                                         //lyxerr << "default creation: m1: " << m << endl;
1698                                         if (at->currentMode() != InsetMath::UNDECIDED_MODE)
1699                                                 m = at->currentMode();
1700                                         //lyxerr << "default creation: m2: " << m << endl;
1701                                         InsetMath::idx_type start = 0;
1702                                         // this fails on \bigg[...\bigg]
1703                                         //MathData opt;
1704                                         //parse(opt, FLAG_OPTION, InsetMath::VERBATIM_MODE);
1705                                         //if (opt.size()) {
1706                                         //      start = 1;
1707                                         //      at.nucleus()->cell(0) = opt;
1708                                         //}
1709                                         for (InsetMath::idx_type i = start; i < at->nargs(); ++i) {
1710                                                 parse(at.nucleus()->cell(i), FLAG_ITEM, m);
1711                                                 skipSpaces();
1712                                         }
1713                                         cell->push_back(at);
1714                                 }
1715                         }
1716                 }
1717
1718
1719                 if (flags & FLAG_LEAVE) {
1720                         flags &= ~FLAG_LEAVE;
1721                         break;
1722                 }
1723         }
1724         return success_;
1725 }
1726
1727
1728
1729 } // anonymous namespace
1730
1731
1732 bool mathed_parse_cell(MathData & ar, docstring const & str, Parse::flags f)
1733 {
1734         return Parser(str, f).parse(ar, 0, f & Parse::TEXTMODE ?
1735                                 InsetMath::TEXT_MODE : InsetMath::MATH_MODE);
1736 }
1737
1738
1739 bool mathed_parse_cell(MathData & ar, istream & is, Parse::flags f)
1740 {
1741         return Parser(is, f).parse(ar, 0, f & Parse::TEXTMODE ?
1742                                 InsetMath::TEXT_MODE : InsetMath::MATH_MODE);
1743 }
1744
1745
1746 bool mathed_parse_normal(MathAtom & t, docstring const & str, Parse::flags f)
1747 {
1748         return Parser(str, f).parse(t);
1749 }
1750
1751
1752 bool mathed_parse_normal(MathAtom & t, Lexer & lex, Parse::flags f)
1753 {
1754         return Parser(lex, f).parse(t);
1755 }
1756
1757
1758 bool mathed_parse_normal(InsetMathGrid & grid, docstring const & str, Parse::flags f)
1759 {
1760         return Parser(str, f).parse1(grid, 0, f & Parse::TEXTMODE ?
1761                         InsetMath::TEXT_MODE : InsetMath::MATH_MODE, false);
1762 }
1763
1764
1765 void initParser()
1766 {
1767         fill(theCatcode, theCatcode + 128, catOther);
1768         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
1769         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
1770
1771         theCatcode[int('\\')] = catEscape;
1772         theCatcode[int('{')]  = catBegin;
1773         theCatcode[int('}')]  = catEnd;
1774         theCatcode[int('$')]  = catMath;
1775         theCatcode[int('&')]  = catAlign;
1776         theCatcode[int('\n')] = catNewline;
1777         theCatcode[int('#')]  = catParameter;
1778         theCatcode[int('^')]  = catSuper;
1779         theCatcode[int('_')]  = catSub;
1780         theCatcode[int(0x7f)] = catIgnore;
1781         theCatcode[int(' ')]  = catSpace;
1782         theCatcode[int('\t')] = catSpace;
1783         theCatcode[int('\r')] = catNewline;
1784         theCatcode[int('~')]  = catActive;
1785         theCatcode[int('%')]  = catComment;
1786 }
1787
1788
1789 } // namespace lyx