src/mathed/MathParser.cpp

   1 /**
   2  * \file MathParser.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  */
  10
  11 /*
  12
  13 If someone desperately needs partial "structures" (such as a few
  14 cells of an array inset or similar) (s)he could uses the
  15 following hack as starting point to write some macros:
  16
  17   \newif\ifcomment
  18   \commentfalse
  19   \ifcomment
  20           \def\makeamptab{\catcode`\&=4\relax}
  21           \def\makeampletter{\catcode`\&=11\relax}
  22     \def\b{\makeampletter\expandafter\makeamptab\bi}
  23     \long\def\bi#1\e{}
  24   \else
  25     \def\b{}\def\e{}
  26   \fi
  27   ...
  28
  29   \[\begin{array}{ccc}
  30 1
  31 &
  32
  33   \end{array}\]
  34
  35 */
  36
  37
  38 #include <config.h>
  39
  40 #include "MathParser.h"
  41
  42 #include "InsetMathArray.h"
  43 #include "InsetMathBig.h"
  44 #include "InsetMathBrace.h"
  45 #include "InsetMathChar.h"
  46 #include "InsetMathColor.h"
  47 #include "InsetMathComment.h"
  48 #include "InsetMathDelim.h"
  49 #include "InsetMathEnsureMath.h"
  50 #include "InsetMathEnv.h"
  51 #include "InsetMathFrac.h"
  52 #include "InsetMathKern.h"
  53 #include "MathMacro.h"
  54 #include "InsetMathPar.h"
  55 #include "InsetMathRef.h"
  56 #include "InsetMathRoot.h"
  57 #include "InsetMathScript.h"
  58 #include "InsetMathSpace.h"
  59 #include "InsetMathSplit.h"
  60 #include "InsetMathSqrt.h"
  61 #include "InsetMathTabular.h"
  62 #include "MathMacroTemplate.h"
  63 #include "MathFactory.h"
  64 #include "MathMacroArgument.h"
  65 #include "MathSupport.h"
  66
  67 #include "Encoding.h"
  68 #include "Lexer.h"
  69
  70 #include "support/debug.h"
  71 #include "support/convert.h"
  72 #include "support/docstream.h"
  73 #include "support/lstrings.h"
  74
  75 #include <sstream>
  76
  77 //#define FILEDEBUG
  78
  79 using namespace std;
  80
  81 namespace lyx {
  82
  83 using support::subst;
  84
  85 namespace {
  86
  87 InsetMath::mode_type asMode(InsetMath::mode_type oldmode, docstring const & str)
  88 {
  89         //lyxerr << "handling mode: '" << str << "'" << endl;
  90         if (str == "mathmode")
  91                 return InsetMath::MATH_MODE;
  92         if (str == "textmode" || str == "forcetext")
  93                 return InsetMath::TEXT_MODE;
  94         return oldmode;
  95 }
  96
  97
  98 bool stared(docstring const & s)
  99 {
 100         size_t const n = s.size();
 101         return n && s[n - 1] == '*';
 102 }
 103
 104
 105 docstring escapeSpecialChars(docstring const & str, bool textmode)
 106 {
 107         docstring const backslash = textmode ? from_ascii("\\textbackslash ")
 108                                              : from_ascii("\\backslash ");
 109         docstring const caret = textmode ? from_ascii("\\textasciicircum ")
 110                                          : from_ascii("\\mathcircumflex ");
 111
 112         return subst(subst(subst(subst(subst(subst(subst(subst(subst(str,
 113                         from_ascii("\\"), backslash),
 114                         from_ascii("^"), caret),
 115                         from_ascii("_"), from_ascii("\\_")),
 116                         from_ascii("$"), from_ascii("\\$")),
 117                         from_ascii("#"), from_ascii("\\#")),
 118                         from_ascii("&"), from_ascii("\\&")),
 119                         from_ascii("%"), from_ascii("\\%")),
 120                         from_ascii("{"), from_ascii("\\{")),
 121                         from_ascii("}"), from_ascii("\\}"));
 122 }
 123
 124
 125 /*!
 126  * Add the row \p cellrow to \p grid.
 127  * \returns wether the row could be added. Adding a row can fail for
 128  * environments like "equation" that have a fixed number of rows.
 129  */
 130 bool addRow(InsetMathGrid & grid, InsetMathGrid::row_type & cellrow,
 131             docstring const & vskip, bool allow_newpage_ = true)
 132 {
 133         ++cellrow;
 134         if (cellrow == grid.nrows()) {
 135                 //lyxerr << "adding row " << cellrow << endl;
 136                 grid.addRow(cellrow - 1);
 137                 if (cellrow == grid.nrows()) {
 138                         // We can't add a row to this grid, so let's
 139                         // append the content of this cell to the previous
 140                         // one.
 141                         // This does not happen in well formed .lyx files,
 142                         // but LyX versions 1.3.x and older could create
 143                         // such files and tex2lyx can still do that.
 144                         --cellrow;
 145                         lyxerr << "ignoring extra row";
 146                         if (!vskip.empty())
 147                                 lyxerr << " with extra space " << to_utf8(vskip);
 148                         if (!allow_newpage_)
 149                                 lyxerr << " with no page break allowed";
 150                         lyxerr << '.' << endl;
 151                         return false;
 152                 }
 153         }
 154         grid.vcrskip(Length(to_utf8(vskip)), cellrow - 1);
 155         grid.rowinfo(cellrow - 1).allow_newpage_ = allow_newpage_;
 156         return true;
 157 }
 158
 159
 160 /*!
 161  * Add the column \p cellcol to \p grid.
 162  * \returns wether the column could be added. Adding a column can fail for
 163  * environments like "eqnarray" that have a fixed number of columns.
 164  */
 165 bool addCol(InsetMathGrid & grid, InsetMathGrid::col_type & cellcol)
 166 {
 167         ++cellcol;
 168         if (cellcol == grid.ncols()) {
 169                 //lyxerr << "adding column " << cellcol << endl;
 170                 grid.addCol(cellcol);
 171                 if (cellcol == grid.ncols()) {
 172                         // We can't add a column to this grid, so let's
 173                         // append the content of this cell to the previous
 174                         // one.
 175                         // This does not happen in well formed .lyx files,
 176                         // but LyX versions 1.3.x and older could create
 177                         // such files and tex2lyx can still do that.
 178                         --cellcol;
 179                         lyxerr << "ignoring extra column." << endl;
 180                         return false;
 181                 }
 182         }
 183         return true;
 184 }
 185
 186
 187 /*!
 188  * Check wether the last row is empty and remove it if yes.
 189  * Otherwise the following code
 190  * \verbatim
 191 \begin{array}{|c|c|}
 192 \hline
 193 1 & 2 \\ \hline
 194 3 & 4 \\ \hline
 195 \end{array}
 196  * \endverbatim
 197  * will result in a grid with 3 rows (+ the dummy row that is always present),
 198  * because the last '\\' opens a new row.
 199  */
 200 void delEmptyLastRow(InsetMathGrid & grid)
 201 {
 202         InsetMathGrid::row_type const row = grid.nrows() - 1;
 203         for (InsetMathGrid::col_type col = 0; col < grid.ncols(); ++col) {
 204                 if (!grid.cell(grid.index(row, col)).empty())
 205                         return;
 206         }
 207         // Copy the row information of the empty row (which would contain the
 208         // last hline in the example above) to the dummy row and delete the
 209         // empty row.
 210         grid.rowinfo(row + 1) = grid.rowinfo(row);
 211         grid.delRow(row);
 212 }
 213
 214
 215 // These are TeX's catcodes
 216 enum CatCode {
 217         catEscape,     // 0    backslash
 218         catBegin,      // 1    {
 219         catEnd,        // 2    }
 220         catMath,       // 3    $
 221         catAlign,      // 4    &
 222         catNewline,    // 5    ^^M
 223         catParameter,  // 6    #
 224         catSuper,      // 7    ^
 225         catSub,        // 8    _
 226         catIgnore,     // 9
 227         catSpace,      // 10   space
 228         catLetter,     // 11   a-zA-Z
 229         catOther,      // 12   none of the above
 230         catActive,     // 13   ~
 231         catComment,    // 14   %
 232         catInvalid     // 15   <delete>
 233 };
 234
 235 CatCode theCatcode[128];
 236
 237
 238 inline CatCode catcode(char_type c)
 239 {
 240         /* The only characters that are not catOther lie in the pure ASCII
 241          * range. Therefore theCatcode has only 128 entries.
 242          * TeX itself deals with 8bit characters, so if needed this table
 243          * could be enlarged to 256 entries.
 244          * Any larger value does not make sense, since the fact that we use
 245          * unicode internally does not change Knuth's TeX engine.
 246          * Apart from that a table for the full 21bit UCS4 range would waste
 247          * too much memory. */
 248         if (c >= 128)
 249                 return catOther;
 250
 251         return theCatcode[c];
 252 }
 253
 254
 255 enum {
 256         FLAG_ALIGN      = 1 << 0,  //  next & or \\ ends the parsing process
 257         FLAG_BRACE_LAST = 1 << 1,  //  next closing brace ends the parsing
 258         FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
 259         FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
 260         FLAG_BRACK_LAST = 1 << 4,  //  next closing bracket ends the parsing
 261         FLAG_TEXTMODE   = 1 << 5,  //  we are in a box
 262         FLAG_ITEM       = 1 << 6,  //  read a (possibly braced) token
 263         FLAG_LEAVE      = 1 << 7,  //  leave the loop at the end
 264         FLAG_SIMPLE     = 1 << 8,  //  next $ leaves the loop
 265         FLAG_EQUATION   = 1 << 9,  //  next \] leaves the loop
 266         FLAG_SIMPLE2    = 1 << 10, //  next \) leaves the loop
 267         FLAG_OPTION     = 1 << 11, //  read [...] style option
 268         FLAG_BRACED     = 1 << 12  //  read {...} style argument
 269 };
 270
 271
 272 //
 273 // Helper class for parsing
 274 //
 275
 276 class Token {
 277 public:
 278         ///
 279         Token() : cs_(), char_(0), cat_(catIgnore) {}
 280         ///
 281         Token(char_type c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
 282         ///
 283         explicit Token(docstring const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
 284
 285         ///
 286         docstring const & cs() const { return cs_; }
 287         ///
 288         CatCode cat() const { return cat_; }
 289         ///
 290         char_type character() const { return char_; }
 291         ///
 292         docstring asString() const { return cs_.size() ? cs_ : docstring(1, char_); }
 293         ///
 294         docstring asInput() const { return cs_.size() ? '\\' + cs_ : docstring(1, char_); }
 295
 296 private:
 297         ///
 298         docstring cs_;
 299         ///
 300         char_type char_;
 301         ///
 302         CatCode cat_;
 303 };
 304
 305
 306 ostream & operator<<(ostream & os, Token const & t)
 307 {
 308         if (t.cs().size()) {
 309                 docstring const & cs = t.cs();
 310                 // FIXME: For some strange reason, the stream operator instanciate
 311                 // a new Token before outputting the contents of t.cs().
 312                 // Because of this the line
 313                 //     os << '\\' << cs;
 314                 // below becomes recursive.
 315                 // In order to avoid that we return early:
 316                 if (cs == "\\")
 317                         return os;
 318                 os << '\\' << to_utf8(cs);
 319         }
 320         else if (t.cat() == catLetter)
 321                 os << t.character();
 322         else
 323                 os << '[' << t.character() << ',' << t.cat() << ']';
 324         return os;
 325 }
 326
 327
 328 class Parser {
 329 public:
 330         ///
 331         typedef  InsetMath::mode_type mode_type;
 332         ///
 333         typedef  Parse::flags parse_mode;
 334
 335         ///
 336         Parser(Lexer & lex, parse_mode mode);
 337         /// Only use this for reading from .lyx file format, for the reason
 338         /// see Parser::tokenize(istream &).
 339         Parser(istream & is, parse_mode mode);
 340         ///
 341         Parser(docstring const & str, parse_mode mode);
 342
 343         ///
 344         bool parse(MathAtom & at);
 345         ///
 346         bool parse(MathData & array, unsigned flags, mode_type mode);
 347         ///
 348         bool parse1(InsetMathGrid & grid, unsigned flags, mode_type mode,
 349                 bool numbered);
 350         ///
 351         MathData parse(unsigned flags, mode_type mode);
 352         ///
 353         int lineno() const { return lineno_; }
 354         ///
 355         void putback();
 356
 357 private:
 358         ///
 359         void parse2(MathAtom & at, unsigned flags, mode_type mode, bool numbered);
 360         /// get arg delimited by 'left' and 'right'
 361         docstring getArg(char_type left, char_type right);
 362         ///
 363         char_type getChar();
 364         ///
 365         void error(string const & msg);
 366         void error(docstring const & msg) { error(to_utf8(msg)); }
 367         /// dump contents to screen
 368         void dump() const;
 369         /// Only use this for reading from .lyx file format (see
 370         /// implementation for reason)
 371         void tokenize(istream & is);
 372         ///
 373         void tokenize(docstring const & s);
 374         ///
 375         void skipSpaceTokens(idocstream & is, char_type c);
 376         ///
 377         void push_back(Token const & t);
 378         ///
 379         void pop_back();
 380         ///
 381         Token const & prevToken() const;
 382         ///
 383         Token const & nextToken() const;
 384         ///
 385         Token const & getToken();
 386         /// skips spaces if any
 387         void skipSpaces();
 388         ///
 389         void lex(docstring const & s);
 390         ///
 391         bool good() const;
 392         ///
 393         docstring parse_verbatim_item();
 394         ///
 395         docstring parse_verbatim_option();
 396
 397         ///
 398         int lineno_;
 399         ///
 400         vector<Token> tokens_;
 401         ///
 402         unsigned pos_;
 403         /// Stack of active environments
 404         vector<docstring> environments_;
 405         ///
 406         parse_mode mode_;
 407         ///
 408         bool success_;
 409 };
 410
 411
 412 Parser::Parser(Lexer & lexer, parse_mode mode)
 413         : lineno_(lexer.lineNumber()), pos_(0), mode_(mode), success_(true)
 414 {
 415         tokenize(lexer.getStream());
 416         lexer.eatLine();
 417 }
 418
 419
 420 Parser::Parser(istream & is, parse_mode mode)
 421         : lineno_(0), pos_(0), mode_(mode), success_(true)
 422 {
 423         tokenize(is);
 424 }
 425
 426
 427 Parser::Parser(docstring const & str, parse_mode mode)
 428         : lineno_(0), pos_(0), mode_(mode), success_(true)
 429 {
 430         tokenize(str);
 431 }
 432
 433
 434 void Parser::push_back(Token const & t)
 435 {
 436         tokens_.push_back(t);
 437 }
 438
 439
 440 void Parser::pop_back()
 441 {
 442         tokens_.pop_back();
 443 }
 444
 445
 446 Token const & Parser::prevToken() const
 447 {
 448         static const Token dummy;
 449         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 450 }
 451
 452
 453 Token const & Parser::nextToken() const
 454 {
 455         static const Token dummy;
 456         return good() ? tokens_[pos_] : dummy;
 457 }
 458
 459
 460 Token const & Parser::getToken()
 461 {
 462         static const Token dummy;
 463         //lyxerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << endl;
 464         return good() ? tokens_[pos_++] : dummy;
 465 }
 466
 467
 468 void Parser::skipSpaces()
 469 {
 470         while (nextToken().cat() == catSpace || nextToken().cat() == catNewline)
 471                 getToken();
 472 }
 473
 474
 475 void Parser::putback()
 476 {
 477         --pos_;
 478 }
 479
 480
 481 bool Parser::good() const
 482 {
 483         return pos_ < tokens_.size();
 484 }
 485
 486
 487 char_type Parser::getChar()
 488 {
 489         if (!good()) {
 490                 error("The input stream is not well...");
 491                 putback();
 492                 return 0;
 493         }
 494         return tokens_[pos_++].character();
 495 }
 496
 497
 498 docstring Parser::getArg(char_type left, char_type right)
 499 {
 500         skipSpaces();
 501
 502         docstring result;
 503         char_type c = getChar();
 504
 505         if (c != left)
 506                 putback();
 507         else
 508                 while ((c = getChar()) != right && good())
 509                         result += c;
 510
 511         return result;
 512 }
 513
 514
 515 void Parser::skipSpaceTokens(idocstream & is, char_type c)
 516 {
 517         // skip trailing spaces
 518         while (catcode(c) == catSpace || catcode(c) == catNewline)
 519                 if (!is.get(c))
 520                         break;
 521         //lyxerr << "putting back: " << c << endl;
 522         is.putback(c);
 523 }
 524
 525
 526 void Parser::tokenize(istream & is)
 527 {
 528         // eat everything up to the next \end_inset or end of stream
 529         // and store it in s for further tokenization
 530         string s;
 531         char c;
 532         while (is.get(c)) {
 533                 s += c;
 534                 if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
 535                         s = s.substr(0, s.size() - 10);
 536                         break;
 537                 }
 538         }
 539         // Remove the space after \end_inset
 540         if (is.get(c) && c != ' ')
 541                 is.unget();
 542
 543         // tokenize buffer
 544         tokenize(from_utf8(s));
 545 }
 546
 547
 548 void Parser::tokenize(docstring const & buffer)
 549 {
 550         idocstringstream is(mode_ & Parse::VERBATIM
 551                         ? escapeSpecialChars(buffer, mode_ & Parse::TEXTMODE)
 552                         : buffer, ios::in | ios::binary);
 553
 554         char_type c;
 555         while (is.get(c)) {
 556                 //lyxerr << "reading c: " << c << endl;
 557
 558                 switch (catcode(c)) {
 559                         case catNewline: {
 560                                 ++lineno_;
 561                                 is.get(c);
 562                                 if (catcode(c) == catNewline)
 563                                         ; //push_back(Token("par"));
 564                                 else {
 565                                         push_back(Token('\n', catNewline));
 566                                         is.putback(c);
 567                                 }
 568                                 break;
 569                         }
 570
 571 /*
 572                         case catComment: {
 573                                 while (is.get(c) && catcode(c) != catNewline)
 574                                         ;
 575                                 ++lineno_;
 576                                 break;
 577                         }
 578 */
 579
 580                         case catEscape: {
 581                                 is.get(c);
 582                                 if (!is) {
 583                                         error("unexpected end of input");
 584                                 } else {
 585                                         docstring s(1, c);
 586                                         if (catcode(c) == catLetter) {
 587                                                 // collect letters
 588                                                 while (is.get(c) && catcode(c) == catLetter)
 589                                                         s += c;
 590                                                 skipSpaceTokens(is, c);
 591                                         }
 592                                         push_back(Token(s));
 593                                 }
 594                                 break;
 595                         }
 596
 597                         case catSuper:
 598                         case catSub: {
 599                                 push_back(Token(c, catcode(c)));
 600                                 is.get(c);
 601                                 skipSpaceTokens(is, c);
 602                                 break;
 603                         }
 604
 605                         case catIgnore: {
 606                                 if (!(mode_ & Parse::QUIET))
 607                                         lyxerr << "ignoring a char: " << int(c) << endl;
 608                                 break;
 609                         }
 610
 611                         default:
 612                                 push_back(Token(c, catcode(c)));
 613                 }
 614         }
 615
 616 #ifdef FILEDEBUG
 617         dump();
 618 #endif
 619 }
 620
 621
 622 void Parser::dump() const
 623 {
 624         lyxerr << "\nTokens: ";
 625         for (unsigned i = 0; i < tokens_.size(); ++i) {
 626                 if (i == pos_)
 627                         lyxerr << " <#> ";
 628                 lyxerr << tokens_[i];
 629         }
 630         lyxerr << " pos: " << pos_ << endl;
 631 }
 632
 633
 634 void Parser::error(string const & msg)
 635 {
 636         success_ = false;
 637         if (!(mode_ & Parse::QUIET)) {
 638                 lyxerr << "Line ~" << lineno_ << ": Math parse error: "
 639                        << msg << endl;
 640                 dump();
 641         }
 642 }
 643
 644
 645 bool Parser::parse(MathAtom & at)
 646 {
 647         skipSpaces();
 648         MathData ar;
 649         parse(ar, false, InsetMath::UNDECIDED_MODE);
 650         if (ar.size() != 1 || ar.front()->getType() == hullNone) {
 651                 if (!(mode_ & Parse::QUIET))
 652                         lyxerr << "unusual contents found: " << ar << endl;
 653                 at = MathAtom(new InsetMathPar(ar));
 654                 //if (at->nargs() > 0)
 655                 //      at.nucleus()->cell(0) = ar;
 656                 //else
 657                 //      lyxerr << "unusual contents found: " << ar << endl;
 658                 success_ = false;
 659         } else
 660                 at = ar[0];
 661         return success_;
 662 }
 663
 664
 665 docstring Parser::parse_verbatim_option()
 666 {
 667         skipSpaces();
 668         docstring res;
 669         if (nextToken().character() == '[') {
 670                 Token t = getToken();
 671                 for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
 672                         if (t.cat() == catBegin) {
 673                                 putback();
 674                                 res += '{' + parse_verbatim_item() + '}';
 675                         } else
 676                                 res += t.asInput();
 677                 }
 678         }
 679         return res;
 680 }
 681
 682
 683 docstring Parser::parse_verbatim_item()
 684 {
 685         skipSpaces();
 686         docstring res;
 687         if (nextToken().cat() == catBegin) {
 688                 Token t = getToken();
 689                 for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
 690                         if (t.cat() == catBegin) {
 691                                 putback();
 692                                 res += '{' + parse_verbatim_item() + '}';
 693                         }
 694                         else
 695                                 res += t.asInput();
 696                 }
 697         }
 698         return res;
 699 }
 700
 701
 702 MathData Parser::parse(unsigned flags, mode_type mode)
 703 {
 704         MathData ar;
 705         parse(ar, flags, mode);
 706         return ar;
 707 }
 708
 709
 710 bool Parser::parse(MathData & array, unsigned flags, mode_type mode)
 711 {
 712         InsetMathGrid grid(1, 1);
 713         parse1(grid, flags, mode, false);
 714         array = grid.cell(0);
 715         return success_;
 716 }
 717
 718
 719 void Parser::parse2(MathAtom & at, const unsigned flags, const mode_type mode,
 720         const bool numbered)
 721 {
 722         parse1(*(at.nucleus()->asGridInset()), flags, mode, numbered);
 723 }
 724
 725
 726 bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
 727         const mode_type mode, const bool numbered)
 728 {
 729         int limits = 0;
 730         InsetMathGrid::row_type cellrow = 0;
 731         InsetMathGrid::col_type cellcol = 0;
 732         MathData * cell = &grid.cell(grid.index(cellrow, cellcol));
 733
 734         if (grid.asHullInset())
 735                 grid.asHullInset()->numbered(cellrow, numbered);
 736
 737         //dump();
 738         //lyxerr << " flags: " << flags << endl;
 739         //lyxerr << " mode: " << mode  << endl;
 740         //lyxerr << "grid: " << grid << endl;
 741
 742         while (good()) {
 743                 Token const & t = getToken();
 744
 745 #ifdef FILEDEBUG
 746                 lyxerr << "t: " << t << " flags: " << flags << endl;
 747                 lyxerr << "mode: " << mode  << endl;
 748                 cell->dump();
 749                 lyxerr << endl;
 750 #endif
 751
 752                 if (flags & FLAG_ITEM) {
 753
 754                         if (t.cat() == catBegin) {
 755                                 // skip the brace and collect everything to the next matching
 756                                 // closing brace
 757                                 parse1(grid, FLAG_BRACE_LAST, mode, numbered);
 758                                 return success_;
 759                         }
 760
 761                         // handle only this single token, leave the loop if done
 762                         flags = FLAG_LEAVE;
 763                 }
 764
 765
 766                 if (flags & FLAG_BRACED) {
 767                         if (t.cat() == catSpace)
 768                                 continue;
 769
 770                         if (t.cat() != catBegin) {
 771                                 error("opening brace expected");
 772                                 return success_;
 773                         }
 774
 775                         // skip the brace and collect everything to the next matching
 776                         // closing brace
 777                         flags = FLAG_BRACE_LAST;
 778                 }
 779
 780
 781                 if (flags & FLAG_OPTION) {
 782                         if (t.cat() == catOther && t.character() == '[') {
 783                                 MathData ar;
 784                                 parse(ar, FLAG_BRACK_LAST, mode);
 785                                 cell->append(ar);
 786                         } else {
 787                                 // no option found, put back token and we are done
 788                                 putback();
 789                         }
 790                         return success_;
 791                 }
 792
 793                 //
 794                 // cat codes
 795                 //
 796                 if (t.cat() == catMath) {
 797                         if (mode != InsetMath::MATH_MODE) {
 798                                 // we are inside some text mode thingy, so opening new math is allowed
 799                                 Token const & n = getToken();
 800                                 if (n.cat() == catMath) {
 801                                         // TeX's $$...$$ syntax for displayed math
 802                                         cell->push_back(MathAtom(new InsetMathHull(hullEquation)));
 803                                         parse2(cell->back(), FLAG_SIMPLE, InsetMath::MATH_MODE, false);
 804                                         getToken(); // skip the second '$' token
 805                                 } else {
 806                                         // simple $...$  stuff
 807                                         putback();
 808                                         if (mode == InsetMath::UNDECIDED_MODE) {
 809                                                 cell->push_back(MathAtom(new InsetMathHull(hullSimple)));
 810                                                 parse2(cell->back(), FLAG_SIMPLE, InsetMath::MATH_MODE, false);
 811                                         } else {
 812                                                 // Don't create nested math hulls (bug #5392)
 813                                                 cell->push_back(MathAtom(new InsetMathEnsureMath));
 814                                                 parse(cell->back().nucleus()->cell(0), FLAG_SIMPLE, InsetMath::MATH_MODE);
 815                                         }
 816                                 }
 817                         }
 818
 819                         else if (flags & FLAG_SIMPLE) {
 820                                 // this is the end of the formula
 821                                 return success_;
 822                         }
 823
 824                         else {
 825                                 error("something strange in the parser");
 826                                 break;
 827                         }
 828                 }
 829
 830                 else if (t.cat() == catLetter)
 831                         cell->push_back(MathAtom(new InsetMathChar(t.character())));
 832
 833                 else if (t.cat() == catSpace && mode != InsetMath::MATH_MODE) {
 834                         if (cell->empty() || cell->back()->getChar() != ' ')
 835                                 cell->push_back(MathAtom(new InsetMathChar(t.character())));
 836                 }
 837
 838                 else if (t.cat() == catNewline && mode != InsetMath::MATH_MODE) {
 839                         if (cell->empty() || cell->back()->getChar() != ' ')
 840                                 cell->push_back(MathAtom(new InsetMathChar(' ')));
 841                 }
 842
 843                 else if (t.cat() == catParameter) {
 844                         Token const & n = getToken();
 845                         cell->push_back(MathAtom(new MathMacroArgument(n.character()-'0')));
 846                 }
 847
 848                 else if (t.cat() == catActive)
 849                         cell->push_back(MathAtom(new InsetMathChar(t.character())));
 850
 851                 else if (t.cat() == catBegin) {
 852                         MathData ar;
 853                         parse(ar, FLAG_BRACE_LAST, mode);
 854                         // do not create a BraceInset if they were written by LyX
 855                         // this helps to keep the annoyance of  "a choose b"  to a minimum
 856                         if (ar.size() == 1 && ar[0]->extraBraces())
 857                                 cell->append(ar);
 858                         else
 859                                 cell->push_back(MathAtom(new InsetMathBrace(ar)));
 860                 }
 861
 862                 else if (t.cat() == catEnd) {
 863                         if (flags & FLAG_BRACE_LAST)
 864                                 return success_;
 865                         error("found '}' unexpectedly");
 866                         //LASSERT(false, /**/);
 867                         //add(cell, '}', LM_TC_TEX);
 868                 }
 869
 870                 else if (t.cat() == catAlign) {
 871                         //lyxerr << " column now " << (cellcol + 1)
 872                         //       << " max: " << grid.ncols() << endl;
 873                         if (flags & FLAG_ALIGN)
 874                                 return success_;
 875                         if (addCol(grid, cellcol))
 876                                 cell = &grid.cell(grid.index(cellrow, cellcol));
 877                 }
 878
 879                 else if (t.cat() == catSuper || t.cat() == catSub) {
 880                         bool up = (t.cat() == catSuper);
 881                         // we need no new script inset if the last thing was a scriptinset,
 882                         // which has that script already not the same script already
 883                         if (!cell->size())
 884                                 cell->push_back(MathAtom(new InsetMathScript(up)));
 885                         else if (cell->back()->asScriptInset() &&
 886                                         !cell->back()->asScriptInset()->has(up))
 887                                 cell->back().nucleus()->asScriptInset()->ensure(up);
 888                         else if (cell->back()->asScriptInset())
 889                                 cell->push_back(MathAtom(new InsetMathScript(up)));
 890                         else
 891                                 cell->back() = MathAtom(new InsetMathScript(cell->back(), up));
 892                         InsetMathScript * p = cell->back().nucleus()->asScriptInset();
 893                         // special handling of {}-bases
 894                         // Here we could remove the brace inset for things
 895                         // like {a'}^2 and add the braces back in
 896                         // InsetMathScript::write().
 897                         // We do not do it, since it is not possible to detect
 898                         // reliably whether the braces are needed because the
 899                         // nucleus contains more than one symbol, or whether
 900                         // they are needed for unknown commands like \xx{a}_0
 901                         // or \yy{a}{b}_0. This was done in revision 14819
 902                         // in an unreliable way. See this thread
 903                         // http://www.mail-archive.com/lyx-devel%40lists.lyx.org/msg104917.html
 904                         // for more details.
 905                         parse(p->cell(p->idxOfScript(up)), FLAG_ITEM, mode);
 906                         if (limits) {
 907                                 p->limits(limits);
 908                                 limits = 0;
 909                         }
 910                 }
 911
 912                 else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
 913                         //lyxerr << "finished reading option" << endl;
 914                         return success_;
 915                 }
 916
 917                 else if (t.cat() == catOther) {
 918                         char_type c = t.character();
 919                         if (isAsciiOrMathAlpha(c)
 920                             || mode_ & Parse::VERBATIM
 921                             || !(mode_ & Parse::USETEXT)
 922                             || mode == InsetMath::TEXT_MODE) {
 923                                 cell->push_back(MathAtom(new InsetMathChar(c)));
 924                         } else {
 925                                 MathAtom at = createInsetMath("text");
 926                                 at.nucleus()->cell(0).push_back(MathAtom(new InsetMathChar(c)));
 927                                 while (nextToken().cat() == catOther
 928                                        && !isAsciiOrMathAlpha(nextToken().character())) {
 929                                         c = getToken().character();
 930                                         at.nucleus()->cell(0).push_back(MathAtom(new InsetMathChar(c)));
 931                                 }
 932                                 cell->push_back(at);
 933                         }
 934                 }
 935
 936                 else if (t.cat() == catComment) {
 937                         docstring s;
 938                         while (good()) {
 939                                 Token const & t = getToken();
 940                                 if (t.cat() == catNewline)
 941                                         break;
 942                                 s += t.asString();
 943                         }
 944                         cell->push_back(MathAtom(new InsetMathComment(s)));
 945                         skipSpaces();
 946                 }
 947
 948                 //
 949                 // control sequences
 950                 //
 951
 952                 else if (t.cs() == "lyxlock") {
 953                         if (cell->size())
 954                                 cell->back().nucleus()->lock(true);
 955                 }
 956
 957                 else if ((t.cs() == "global" && nextToken().cs() == "def") ||
 958                          t.cs() == "def") {
 959                         if (t.cs() == "global")
 960                                 getToken();
 961
 962                         // get name
 963                         docstring name = getToken().cs();
 964
 965                         // read parameters
 966                         int nargs = 0;
 967                         docstring pars;
 968                         while (good() && nextToken().cat() != catBegin) {
 969                                 pars += getToken().cs();
 970                                 ++nargs;
 971                         }
 972                         nargs /= 2;
 973
 974                         // read definition
 975                         MathData def;
 976                         parse(def, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
 977
 978                         // is a version for display attached?
 979                         skipSpaces();
 980                         MathData display;
 981                         if (nextToken().cat() == catBegin)
 982                                 parse(display, FLAG_ITEM, InsetMath::MATH_MODE);
 983
 984                         cell->push_back(MathAtom(new MathMacroTemplate(name, nargs,
 985                                0, MacroTypeDef, vector<MathData>(), def, display)));
 986                 }
 987
 988                 else if (t.cs() == "newcommand" ||
 989                          t.cs() == "renewcommand" ||
 990                          t.cs() == "newlyxcommand") {
 991                         // get name
 992                         if (getToken().cat() != catBegin) {
 993                                 error("'{' in \\newcommand expected (1) ");
 994                                 return success_;
 995                         }
 996                         docstring name = getToken().cs();
 997                         if (getToken().cat() != catEnd) {
 998                                 error("'}' in \\newcommand expected");
 999                                 return success_;
1000                         }
1001
1002                         // get arity
1003                         docstring const arg = getArg('[', ']');
1004                         int nargs = 0;
1005                         if (!arg.empty())
1006                                 nargs = convert<int>(arg);
1007
1008                         // optional argument given?
1009                         skipSpaces();
1010                         int optionals = 0;
1011                         vector<MathData> optionalValues;
1012                         while (nextToken().character() == '[') {
1013                                 getToken();
1014                                 optionalValues.push_back(MathData());
1015                                 parse(optionalValues[optionals], FLAG_BRACK_LAST, mode);
1016                                 ++optionals;
1017                         }
1018
1019                         MathData def;
1020                         parse(def, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
1021
1022                         // is a version for display attached?
1023                         skipSpaces();
1024                         MathData display;
1025                         if (nextToken().cat() == catBegin)
1026                                 parse(display, FLAG_ITEM, InsetMath::MATH_MODE);
1027
1028                         cell->push_back(MathAtom(new MathMacroTemplate(name, nargs,
1029                                 optionals, MacroTypeNewcommand, optionalValues, def, display)));
1030
1031                 }
1032
1033                 else if (t.cs() == "newcommandx" ||
1034                          t.cs() == "renewcommandx") {
1035                         // \newcommandx{\foo}[2][usedefault, addprefix=\global,1=default]{#1,#2}
1036                         // get name
1037                         docstring name;
1038                         if (nextToken().cat() == catBegin) {
1039                                 getToken();
1040                                 name = getToken().cs();
1041                                 if (getToken().cat() != catEnd) {
1042                                         error("'}' in \\newcommandx expected");
1043                                         return success_;
1044                                 }
1045                         } else
1046                                 name = getToken().cs();
1047
1048                         // get arity
1049                         docstring const arg = getArg('[', ']');
1050                         if (arg.empty()) {
1051                                 error("[num] in \\newcommandx expected");
1052                                 return success_;
1053                         }
1054                         int nargs = convert<int>(arg);
1055
1056                         // get options
1057                         int optionals = 0;
1058                         vector<MathData> optionalValues;
1059                         if (nextToken().character() == '[') {
1060                                 // skip '['
1061                                 getToken();
1062
1063                                 // handle 'opt=value' options, separated by ','.
1064                                 skipSpaces();
1065                                 while (nextToken().character() != ']' && good()) {
1066                                         if (nextToken().character() >= '1'
1067                                             && nextToken().character() <= '9') {
1068                                                 // optional value -> get parameter number
1069                                                 int n = getChar() - '0';
1070                                                 if (n > nargs) {
1071                                                         error("Arity of \\newcommandx too low "
1072                                                               "for given optional parameter.");
1073                                                         return success_;
1074                                                 }
1075
1076                                                 // skip '='
1077                                                 if (getToken().character() != '=') {
1078                                                         error("'=' and optional parameter value "
1079                                                               "expected for \\newcommandx");
1080                                                         return success_;
1081                                                 }
1082
1083                                                 // get value
1084                                                 int optNum = max(size_t(n), optionalValues.size());
1085                                                 optionalValues.resize(optNum);
1086                                                 optionalValues[n - 1].clear();
1087                                                 while (nextToken().character() != ']'
1088                                                        && nextToken().character() != ',') {
1089                                                         MathData data;
1090                                                         parse(data, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
1091                                                         optionalValues[n - 1].append(data);
1092                                                 }
1093                                                 optionals = max(n, optionals);
1094                                         } else if (nextToken().cat() == catLetter) {
1095                                                 // we in fact ignore every non-optional
1096                                                 // parameter
1097
1098                                                 // get option name
1099                                                 docstring opt;
1100                                                 while (nextToken().cat() == catLetter)
1101                                                         opt += getChar();
1102
1103                                                 // value?
1104                                                 skipSpaces();
1105                                                 MathData value;
1106                                                 if (nextToken().character() == '=') {
1107                                                         getToken();
1108                                                         while (nextToken().character() != ']'
1109                                                                 && nextToken().character() != ',')
1110                                                                 parse(value, FLAG_ITEM,
1111                                                                       InsetMath::UNDECIDED_MODE);
1112                                                 }
1113                                         } else {
1114                                                 error("option for \\newcommandx expected");
1115                                                 return success_;
1116                                         }
1117
1118                                         // skip komma
1119                                         skipSpaces();
1120                                         if (nextToken().character() == ',') {
1121                                                 getChar();
1122                                                 skipSpaces();
1123                                         } else if (nextToken().character() != ']') {
1124                                                 error("Expecting ',' or ']' in options "
1125                                                       "of \\newcommandx");
1126                                                 return success_;
1127                                         }
1128                                 }
1129
1130                                 // skip ']'
1131                                 if (!good())
1132                                         return success_;
1133                                 getToken();
1134                         }
1135
1136                         // get definition
1137                         MathData def;
1138                         parse(def, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
1139
1140                         // is a version for display attached?
1141                         skipSpaces();
1142                         MathData display;
1143                         if (nextToken().cat() == catBegin)
1144                                 parse(display, FLAG_ITEM, InsetMath::MATH_MODE);
1145
1146                         cell->push_back(MathAtom(new MathMacroTemplate(name, nargs,
1147                                 optionals, MacroTypeNewcommandx, optionalValues, def,
1148                                 display)));
1149                 }
1150
1151                 else if (t.cs() == "(") {
1152                         if (mode == InsetMath::MATH_MODE) {
1153                                 error("bad math environment");
1154                                 break;
1155                         }
1156                         cell->push_back(MathAtom(new InsetMathHull(hullSimple)));
1157                         parse2(cell->back(), FLAG_SIMPLE2, InsetMath::MATH_MODE, false);
1158                 }
1159
1160                 else if (t.cs() == "[") {
1161                         if (mode != InsetMath::UNDECIDED_MODE) {
1162                                 error("bad math environment");
1163                                 break;
1164                         }
1165                         cell->push_back(MathAtom(new InsetMathHull(hullEquation)));
1166                         parse2(cell->back(), FLAG_EQUATION, InsetMath::MATH_MODE, false);
1167                 }
1168
1169                 else if (t.cs() == "protect")
1170                         // ignore \\protect, will hopefully be re-added during output
1171                         ;
1172
1173                 else if (t.cs() == "end") {
1174                         if (flags & FLAG_END) {
1175                                 // eat environment name
1176                                 docstring const name = getArg('{', '}');
1177                                 if (environments_.empty())
1178                                         error("'found \\end{" + name +
1179                                               "}' without matching '\\begin{" +
1180                                               name + "}'");
1181                                 else if (name != environments_.back())
1182                                         error("'\\end{" + name +
1183                                               "}' does not match '\\begin{" +
1184                                               environments_.back() + "}'");
1185                                 else {
1186                                         environments_.pop_back();
1187                                         // Delete empty last row in matrix
1188                                         // like insets.
1189                                         // If you abuse InsetMathGrid for
1190                                         // non-matrix like structures you
1191                                         // probably need to refine this test.
1192                                         // Right now we only have to test for
1193                                         // single line hull insets.
1194                                         if (grid.nrows() > 1)
1195                                                 delEmptyLastRow(grid);
1196                                         return success_;
1197                                 }
1198                         } else
1199                                 error("found 'end' unexpectedly");
1200                 }
1201
1202                 else if (t.cs() == ")") {
1203                         if (flags & FLAG_SIMPLE2)
1204                                 return success_;
1205                         error("found '\\)' unexpectedly");
1206                 }
1207
1208                 else if (t.cs() == "]") {
1209                         if (flags & FLAG_EQUATION)
1210                                 return success_;
1211                         error("found '\\]' unexpectedly");
1212                 }
1213
1214                 else if (t.cs() == "\\") {
1215                         if (flags & FLAG_ALIGN)
1216                                 return success_;
1217                         bool added = false;
1218                         if (nextToken().asInput() == "*") {
1219                                 getToken();
1220                                 added = addRow(grid, cellrow, docstring(), false);
1221                         } else if (good())
1222                                 added = addRow(grid, cellrow, getArg('[', ']'));
1223                         else
1224                                 error("missing token after \\\\");
1225                         if (added) {
1226                                 cellcol = 0;
1227                                 if (grid.asHullInset())
1228                                         grid.asHullInset()->numbered(
1229                                                         cellrow, numbered);
1230                                 cell = &grid.cell(grid.index(cellrow,
1231                                                              cellcol));
1232                         }
1233                 }
1234
1235 #if 0
1236                 else if (t.cs() == "multicolumn") {
1237                         // extract column count and insert dummy cells
1238                         MathData count;
1239                         parse(count, FLAG_ITEM, mode);
1240                         int cols = 1;
1241                         if (!extractNumber(count, cols)) {
1242                                 success_ = false;
1243                                 lyxerr << " can't extract number of cells from " << count << endl;
1244                         }
1245                         // resize the table if necessary
1246                         for (int i = 0; i < cols; ++i) {
1247                                 if (addCol(grid, cellcol)) {
1248                                         cell = &grid.cell(grid.index(
1249                                                         cellrow, cellcol));
1250                                         // mark this as dummy
1251                                         grid.cellinfo(grid.index(
1252                                                 cellrow, cellcol)).dummy_ = true;
1253                                 }
1254                         }
1255                         // the last cell is the real thing, not a dummy
1256                         grid.cellinfo(grid.index(cellrow, cellcol)).dummy_ = false;
1257
1258                         // read special alignment
1259                         MathData align;
1260                         parse(align, FLAG_ITEM, mode);
1261                         //grid.cellinfo(grid.index(cellrow, cellcol)).align_ = extractString(align);
1262
1263                         // parse the remaining contents into the "real" cell
1264                         parse(*cell, FLAG_ITEM, mode);
1265                 }
1266 #endif
1267
1268                 else if (t.cs() == "limits")
1269                         limits = 1;
1270
1271                 else if (t.cs() == "nolimits")
1272                         limits = -1;
1273
1274                 else if (t.cs() == "nonumber") {
1275                         if (grid.asHullInset())
1276                                 grid.asHullInset()->numbered(cellrow, false);
1277                 }
1278
1279                 else if (t.cs() == "number") {
1280                         if (grid.asHullInset())
1281                                 grid.asHullInset()->numbered(cellrow, true);
1282                 }
1283
1284                 else if (t.cs() == "hline") {
1285                         grid.rowinfo(cellrow).lines_ ++;
1286                 }
1287
1288                 else if (t.cs() == "sqrt") {
1289                         MathData ar;
1290                         parse(ar, FLAG_OPTION, mode);
1291                         if (ar.size()) {
1292                                 cell->push_back(MathAtom(new InsetMathRoot));
1293                                 cell->back().nucleus()->cell(0) = ar;
1294                                 parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
1295                         } else {
1296                                 cell->push_back(MathAtom(new InsetMathSqrt));
1297                                 parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1298                         }
1299                 }
1300
1301                 else if (t.cs() == "unit") {
1302                         // Allowed formats \unit[val]{unit}
1303                         MathData ar;
1304                         parse(ar, FLAG_OPTION, mode);
1305                         if (ar.size()) {
1306                                 cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::UNIT)));
1307                                 cell->back().nucleus()->cell(0) = ar;
1308                                 parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
1309                         } else {
1310                                 cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::UNIT, 1)));
1311                                 parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1312                         }
1313                 }
1314
1315                 else if (t.cs() == "unitfrac") {
1316                         // Here allowed formats are \unitfrac[val]{num}{denom}
1317                         MathData ar;
1318                         parse(ar, FLAG_OPTION, mode);
1319                         if (ar.size()) {
1320                                 cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::UNITFRAC, 3)));
1321                                 cell->back().nucleus()->cell(2) = ar;
1322                         } else {
1323                                 cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::UNITFRAC)));
1324                         }
1325                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1326                         parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
1327                 }
1328
1329                 else if (t.cs() == "cfrac") {
1330                         // allowed formats are \cfrac[pos]{num}{denom}
1331                         docstring const arg = getArg('[', ']');
1332                         //lyxerr << "got so far: '" << arg << "'" << endl;
1333                                 if (arg == "l")
1334                                         cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::CFRACLEFT)));
1335                                 else if (arg == "r")
1336                                         cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::CFRACRIGHT)));
1337                                 else if (arg.empty() || arg == "c")
1338                                         cell->push_back(MathAtom(new InsetMathFrac(InsetMathFrac::CFRAC)));
1339                                 else {
1340                                         error("found invalid optional argument");
1341                                         break;
1342                                 }
1343                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1344                         parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
1345                 }
1346
1347                 else if (t.cs() == "xrightarrow" || t.cs() == "xleftarrow") {
1348                         cell->push_back(createInsetMath(t.cs()));
1349                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, mode);
1350                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1351                 }
1352
1353                 else if (t.cs() == "ref" || t.cs() == "eqref" || t.cs() == "prettyref"
1354                           || t.cs() == "pageref" || t.cs() == "vpageref" || t.cs() == "vref") {
1355                         cell->push_back(MathAtom(new InsetMathRef(t.cs())));
1356                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, mode);
1357                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1358                 }
1359
1360                 else if (t.cs() == "left") {
1361                         skipSpaces();
1362                         Token const & tl = getToken();
1363                         // \| and \Vert are equivalent, and InsetMathDelim
1364                         // can't handle \|
1365                         // FIXME: fix this in InsetMathDelim itself!
1366                         docstring const l = tl.cs() == "|" ? from_ascii("Vert") : tl.asString();
1367                         MathData ar;
1368                         parse(ar, FLAG_RIGHT, mode);
1369                         if (!good())
1370                                 break;
1371                         skipSpaces();
1372                         Token const & tr = getToken();
1373                         docstring const r = tr.cs() == "|" ? from_ascii("Vert") : tr.asString();
1374                         cell->push_back(MathAtom(new InsetMathDelim(l, r, ar)));
1375                 }
1376
1377                 else if (t.cs() == "right") {
1378                         if (flags & FLAG_RIGHT)
1379                                 return success_;
1380                         //lyxerr << "got so far: '" << cell << "'" << endl;
1381                         error("Unmatched right delimiter");
1382                         return success_;
1383                 }
1384
1385                 else if (t.cs() == "begin") {
1386                         docstring const name = getArg('{', '}');
1387                         environments_.push_back(name);
1388
1389                         if (name == "array" || name == "subarray") {
1390                                 docstring const valign = parse_verbatim_option() + 'c';
1391                                 docstring const halign = parse_verbatim_item();
1392                                 cell->push_back(MathAtom(new InsetMathArray(name,
1393                                         InsetMathGrid::guessColumns(halign), 1, (char)valign[0], halign)));
1394                                 parse2(cell->back(), FLAG_END, mode, false);
1395                         }
1396
1397                         else if (name == "tabular") {
1398                                 docstring const valign = parse_verbatim_option() + 'c';
1399                                 docstring const halign = parse_verbatim_item();
1400                                 cell->push_back(MathAtom(new InsetMathTabular(name,
1401                                         InsetMathGrid::guessColumns(halign), 1, (char)valign[0], halign)));
1402                                 parse2(cell->back(), FLAG_END, InsetMath::TEXT_MODE, false);
1403                         }
1404
1405                         else if (name == "split" || name == "cases") {
1406                                 cell->push_back(createInsetMath(name));
1407                                 parse2(cell->back(), FLAG_END, mode, false);
1408                         }
1409
1410                         else if (name == "alignedat") {
1411                                 docstring const valign = parse_verbatim_option() + 'c';
1412                                 // ignore this for a while
1413                                 getArg('{', '}');
1414                                 cell->push_back(MathAtom(new InsetMathSplit(name, (char)valign[0])));
1415                                 parse2(cell->back(), FLAG_END, mode, false);
1416                         }
1417
1418                         else if (name == "math") {
1419                                 if (mode == InsetMath::MATH_MODE) {
1420                                         error("bad math environment");
1421                                         break;
1422                                 }
1423                                 cell->push_back(MathAtom(new InsetMathHull(hullSimple)));
1424                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, true);
1425                         }
1426
1427                         else if (name == "equation" || name == "equation*"
1428                                         || name == "displaymath") {
1429                                 if (mode != InsetMath::UNDECIDED_MODE) {
1430                                         error("bad math environment");
1431                                         break;
1432                                 }
1433                                 cell->push_back(MathAtom(new InsetMathHull(hullEquation)));
1434                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, (name == "equation"));
1435                         }
1436
1437                         else if (name == "eqnarray" || name == "eqnarray*") {
1438                                 if (mode != InsetMath::UNDECIDED_MODE) {
1439                                         error("bad math environment");
1440                                         break;
1441                                 }
1442                                 cell->push_back(MathAtom(new InsetMathHull(hullEqnArray)));
1443                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1444                         }
1445
1446                         else if (name == "align" || name == "align*") {
1447                                 if (mode != InsetMath::UNDECIDED_MODE) {
1448                                         error("bad math environment");
1449                                         break;
1450                                 }
1451                                 cell->push_back(MathAtom(new InsetMathHull(hullAlign)));
1452                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1453                         }
1454
1455                         else if (name == "flalign" || name == "flalign*") {
1456                                 if (mode != InsetMath::UNDECIDED_MODE) {
1457                                         error("bad math environment");
1458                                         break;
1459                                 }
1460                                 cell->push_back(MathAtom(new InsetMathHull(hullFlAlign)));
1461                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1462                         }
1463
1464                         else if (name == "alignat" || name == "alignat*") {
1465                                 if (mode != InsetMath::UNDECIDED_MODE) {
1466                                         error("bad math environment");
1467                                         break;
1468                                 }
1469                                 // ignore this for a while
1470                                 getArg('{', '}');
1471                                 cell->push_back(MathAtom(new InsetMathHull(hullAlignAt)));
1472                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1473                         }
1474
1475                         else if (name == "xalignat" || name == "xalignat*") {
1476                                 if (mode != InsetMath::UNDECIDED_MODE) {
1477                                         error("bad math environment");
1478                                         break;
1479                                 }
1480                                 // ignore this for a while
1481                                 getArg('{', '}');
1482                                 cell->push_back(MathAtom(new InsetMathHull(hullXAlignAt)));
1483                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1484                         }
1485
1486                         else if (name == "xxalignat") {
1487                                 if (mode != InsetMath::UNDECIDED_MODE) {
1488                                         error("bad math environment");
1489                                         break;
1490                                 }
1491                                 // ignore this for a while
1492                                 getArg('{', '}');
1493                                 cell->push_back(MathAtom(new InsetMathHull(hullXXAlignAt)));
1494                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1495                         }
1496
1497                         else if (name == "multline" || name == "multline*") {
1498                                 if (mode != InsetMath::UNDECIDED_MODE) {
1499                                         error("bad math environment");
1500                                         break;
1501                                 }
1502                                 cell->push_back(MathAtom(new InsetMathHull(hullMultline)));
1503                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1504                         }
1505
1506                         else if (name == "gather" || name == "gather*") {
1507                                 if (mode != InsetMath::UNDECIDED_MODE) {
1508                                         error("bad math environment");
1509                                         break;
1510                                 }
1511                                 cell->push_back(MathAtom(new InsetMathHull(hullGather)));
1512                                 parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, !stared(name));
1513                         }
1514
1515                         else if (latexkeys const * l = in_word_set(name)) {
1516                                 if (l->inset == "matrix") {
1517                                         cell->push_back(createInsetMath(name));
1518                                         parse2(cell->back(), FLAG_END, mode, false);
1519                                 } else if (l->inset == "split") {
1520                                         docstring const valign = parse_verbatim_option() + 'c';
1521                                         cell->push_back(MathAtom(new InsetMathSplit(name, (char)valign[0])));
1522                                         parse2(cell->back(), FLAG_END, mode, false);
1523                                 } else {
1524                                         success_ = false;
1525                                         if (!(mode_ & Parse::QUIET)) {
1526                                                 dump();
1527                                                 lyxerr << "found math environment `"
1528                                                        << to_utf8(name)
1529                                                        << "' in symbols file with unsupported inset `"
1530                                                        << to_utf8(l->inset)
1531                                                        << "'." << endl;
1532                                         }
1533                                         // create generic environment inset
1534                                         cell->push_back(MathAtom(new InsetMathEnv(name)));
1535                                         parse(cell->back().nucleus()->cell(0), FLAG_END, mode);
1536                                 }
1537                         }
1538
1539                         else {
1540                                 success_ = false;
1541                                 if (!(mode_ & Parse::QUIET)) {
1542                                         dump();
1543                                         lyxerr << "found unknown math environment '"
1544                                                << to_utf8(name) << "'" << endl;
1545                                 }
1546                                 // create generic environment inset
1547                                 cell->push_back(MathAtom(new InsetMathEnv(name)));
1548                                 parse(cell->back().nucleus()->cell(0), FLAG_END, mode);
1549                         }
1550                 }
1551
1552                 else if (t.cs() == "kern") {
1553                         // FIXME: A hack...
1554                         docstring s;
1555                         while (true) {
1556                                 Token const & t = getToken();
1557                                 if (!good()) {
1558                                         putback();
1559                                         break;
1560                                 }
1561                                 s += t.character();
1562                                 if (isValidLength(to_utf8(s)))
1563                                         break;
1564                         }
1565                         cell->push_back(MathAtom(new InsetMathKern(s)));
1566                 }
1567
1568                 else if (t.cs() == "label") {
1569                         // FIXME: This is swallowed in inline formulas
1570                         docstring label = parse_verbatim_item();
1571                         MathData ar;
1572                         asArray(label, ar);
1573                         if (grid.asHullInset()) {
1574                                 grid.asHullInset()->label(cellrow, label);
1575                         } else {
1576                                 cell->push_back(createInsetMath(t.cs()));
1577                                 cell->push_back(MathAtom(new InsetMathBrace(ar)));
1578                         }
1579                 }
1580
1581                 else if (t.cs() == "choose" || t.cs() == "over"
1582                                 || t.cs() == "atop" || t.cs() == "brace"
1583                                 || t.cs() == "brack") {
1584                         MathAtom at = createInsetMath(t.cs());
1585                         at.nucleus()->cell(0) = *cell;
1586                         cell->clear();
1587                         parse(at.nucleus()->cell(1), flags, mode);
1588                         cell->push_back(at);
1589                         return success_;
1590                 }
1591
1592                 else if (t.cs() == "color") {
1593                         docstring const color = parse_verbatim_item();
1594                         cell->push_back(MathAtom(new InsetMathColor(true, color)));
1595                         parse(cell->back().nucleus()->cell(0), flags, mode);
1596                         return success_;
1597                 }
1598
1599                 else if (t.cs() == "textcolor") {
1600                         docstring const color = parse_verbatim_item();
1601                         cell->push_back(MathAtom(new InsetMathColor(false, color)));
1602                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, InsetMath::TEXT_MODE);
1603                 }
1604
1605                 else if (t.cs() == "normalcolor") {
1606                         cell->push_back(createInsetMath(t.cs()));
1607                         parse(cell->back().nucleus()->cell(0), flags, mode);
1608                         return success_;
1609                 }
1610
1611                 else if (t.cs() == "substack") {
1612                         cell->push_back(createInsetMath(t.cs()));
1613                         parse2(cell->back(), FLAG_ITEM, mode, false);
1614                 }
1615
1616                 else if (t.cs() == "xymatrix") {
1617                         odocstringstream os;
1618                         while (good() && nextToken().cat() != catBegin)
1619                                 os << getToken().asInput();
1620                         cell->push_back(createInsetMath(t.cs() + os.str()));
1621                         parse2(cell->back(), FLAG_ITEM, mode, false);
1622                 }
1623
1624                 else if (t.cs() == "framebox" || t.cs() == "makebox") {
1625                         cell->push_back(createInsetMath(t.cs()));
1626                         parse(cell->back().nucleus()->cell(0), FLAG_OPTION, InsetMath::TEXT_MODE);
1627                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, InsetMath::TEXT_MODE);
1628                         parse(cell->back().nucleus()->cell(2), FLAG_ITEM, InsetMath::TEXT_MODE);
1629                 }
1630
1631                 else if (t.cs() == "tag") {
1632                         if (nextToken().character() == '*') {
1633                                 getToken();
1634                                 cell->push_back(createInsetMath(t.cs() + '*'));
1635                         } else
1636                                 cell->push_back(createInsetMath(t.cs()));
1637                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, InsetMath::TEXT_MODE);
1638                 }
1639
1640                 else if (t.cs() == "hspace" && nextToken().character() != '*') {
1641                         docstring const name = t.cs();
1642                         docstring const arg = parse_verbatim_item();
1643                         Length length;
1644                         if (isValidLength(to_utf8(arg), &length))
1645                                 cell->push_back(MathAtom(new InsetMathSpace(length)));
1646                         else {
1647                                 // Since the Length class cannot use length variables
1648                                 // we must not create an InsetMathSpace.
1649                                 cell->push_back(MathAtom(new MathMacro(name)));
1650                                 MathData ar;
1651                                 mathed_parse_cell(ar, '{' + arg + '}');
1652                                 cell->append(ar);
1653                         }
1654                 }
1655
1656 #if 0
1657                 else if (t.cs() == "infer") {
1658                         MathData ar;
1659                         parse(ar, FLAG_OPTION, mode);
1660                         cell->push_back(createInsetMath(t.cs()));
1661                         parse2(cell->back(), FLAG_ITEM, mode, false);
1662                 }
1663
1664                 // Disabled
1665                 else if (1 && t.cs() == "ar") {
1666                         auto_ptr<InsetMathXYArrow> p(new InsetMathXYArrow);
1667                         // try to read target
1668                         parse(p->cell(0), FLAG_OTPTION, mode);
1669                         // try to read label
1670                         if (nextToken().cat() == catSuper || nextToken().cat() == catSub) {
1671                                 p->up_ = nextToken().cat() == catSuper;
1672                                 getToken();
1673                                 parse(p->cell(1), FLAG_ITEM, mode);
1674                                 //lyxerr << "read label: " << p->cell(1) << endl;
1675                         }
1676
1677                         cell->push_back(MathAtom(p.release()));
1678                         //lyxerr << "read cell: " << cell << endl;
1679                 }
1680 #endif
1681
1682                 else if (t.cs() == "lyxmathsym") {
1683                         skipSpaces();
1684                         if (getToken().cat() != catBegin) {
1685                                 error("'{' expected in \\" + t.cs());
1686                                 return success_;
1687                         }
1688                         int count = 0;
1689                         docstring cmd;
1690                         CatCode cat = nextToken().cat();
1691                         while (good() && (count || cat != catEnd)) {
1692                                 if (cat == catBegin)
1693                                         ++count;
1694                                 else if (cat == catEnd)
1695                                         --count;
1696                                 cmd += getToken().asInput();
1697                                 cat = nextToken().cat();
1698                         }
1699                         if (getToken().cat() != catEnd) {
1700                                 error("'}' expected in \\" + t.cs());
1701                                 return success_;
1702                         }
1703                         docstring rem;
1704                         do {
1705                                 cmd = Encodings::fromLaTeXCommand(cmd, rem);
1706                                 for (size_t i = 0; i < cmd.size(); ++i)
1707                                         cell->push_back(MathAtom(new InsetMathChar(cmd[i])));
1708                                 if (rem.size()) {
1709                                         char_type c = rem[0];
1710                                         cell->push_back(MathAtom(new InsetMathChar(c)));
1711                                         cmd = rem.substr(1);
1712                                         rem.clear();
1713                                 } else
1714                                         cmd.clear();
1715                         } while (cmd.size());
1716                 }
1717
1718                 else if (t.cs().size()) {
1719                         latexkeys const * l = in_word_set(t.cs());
1720                         if (l) {
1721                                 if (l->inset == "big") {
1722                                         skipSpaces();
1723                                         docstring const delim = getToken().asInput();
1724                                         if (InsetMathBig::isBigInsetDelim(delim))
1725                                                 cell->push_back(MathAtom(
1726                                                         new InsetMathBig(t.cs(), delim)));
1727                                         else {
1728                                                 cell->push_back(createInsetMath(t.cs()));
1729                                                 putback();
1730                                         }
1731                                 }
1732
1733                                 else if (l->inset == "font") {
1734                                         cell->push_back(createInsetMath(t.cs()));
1735                                         parse(cell->back().nucleus()->cell(0),
1736                                                 FLAG_ITEM, asMode(mode, l->extra));
1737                                 }
1738
1739                                 else if (l->inset == "oldfont") {
1740                                         cell->push_back(createInsetMath(t.cs()));
1741                                         parse(cell->back().nucleus()->cell(0),
1742                                                 flags | FLAG_ALIGN, asMode(mode, l->extra));
1743                                         if (prevToken().cat() != catAlign &&
1744                                             prevToken().cs() != "\\")
1745                                                 return success_;
1746                                         putback();
1747                                 }
1748
1749                                 else if (l->inset == "style") {
1750                                         cell->push_back(createInsetMath(t.cs()));
1751                                         parse(cell->back().nucleus()->cell(0),
1752                                                 flags | FLAG_ALIGN, mode);
1753                                         if (prevToken().cat() != catAlign &&
1754                                             prevToken().cs() != "\\")
1755                                                 return success_;
1756                                         putback();
1757                                 }
1758
1759                                 else {
1760                                         MathAtom at = createInsetMath(t.cs());
1761                                         for (InsetMath::idx_type i = 0; i < at->nargs(); ++i)
1762                                                 parse(at.nucleus()->cell(i),
1763                                                         FLAG_ITEM, asMode(mode, l->extra));
1764                                         cell->push_back(at);
1765                                 }
1766                         }
1767
1768                         else {
1769                                 bool is_unicode_symbol = false;
1770                                 if (mode == InsetMath::TEXT_MODE) {
1771                                         int num_tokens = 0;
1772                                         docstring cmd = prevToken().asInput();
1773                                         skipSpaces();
1774                                         CatCode cat = nextToken().cat();
1775                                         if (cat == catBegin) {
1776                                                 int count = 0;
1777                                                 while (good() && (count || cat != catEnd)) {
1778                                                         cat = nextToken().cat();
1779                                                         cmd += getToken().asInput();
1780                                                         ++num_tokens;
1781                                                         if (cat == catBegin)
1782                                                                 ++count;
1783                                                         else if (cat == catEnd)
1784                                                                 --count;
1785                                                 }
1786                                         }
1787                                         bool is_combining;
1788                                         char_type c =
1789                                                 Encodings::fromLaTeXCommand(cmd, is_combining);
1790                                         if (is_combining) {
1791                                                 if (cat == catLetter)
1792                                                         cmd += '{';
1793                                                 cmd += getToken().asInput();
1794                                                 ++num_tokens;
1795                                                 if (cat == catLetter)
1796                                                         cmd += '}';
1797                                                 c = Encodings::fromLaTeXCommand(cmd, is_combining);
1798                                         }
1799                                         if (c) {
1800                                                 is_unicode_symbol = true;
1801                                                 cell->push_back(MathAtom(new InsetMathChar(c)));
1802                                         } else {
1803                                                 while (num_tokens--)
1804                                                         putback();
1805                                         }
1806                                 }
1807                                 if (!is_unicode_symbol) {
1808                                         MathAtom at = createInsetMath(t.cs());
1809                                         InsetMath::mode_type m = mode;
1810                                         //if (m == InsetMath::UNDECIDED_MODE)
1811                                         //lyxerr << "default creation: m1: " << m << endl;
1812                                         if (at->currentMode() != InsetMath::UNDECIDED_MODE)
1813                                                 m = at->currentMode();
1814                                         //lyxerr << "default creation: m2: " << m << endl;
1815                                         InsetMath::idx_type start = 0;
1816                                         // this fails on \bigg[...\bigg]
1817                                         //MathData opt;
1818                                         //parse(opt, FLAG_OPTION, InsetMath::VERBATIM_MODE);
1819                                         //if (opt.size()) {
1820                                         //      start = 1;
1821                                         //      at.nucleus()->cell(0) = opt;
1822                                         //}
1823                                         for (InsetMath::idx_type i = start; i < at->nargs(); ++i) {
1824                                                 parse(at.nucleus()->cell(i), FLAG_ITEM, m);
1825                                                 skipSpaces();
1826                                         }
1827                                         cell->push_back(at);
1828                                 }
1829                         }
1830                 }
1831
1832
1833                 if (flags & FLAG_LEAVE) {
1834                         flags &= ~FLAG_LEAVE;
1835                         break;
1836                 }
1837         }
1838         return success_;
1839 }
1840
1841
1842
1843 } // anonymous namespace
1844
1845
1846 bool mathed_parse_cell(MathData & ar, docstring const & str, Parse::flags f)
1847 {
1848         return Parser(str, f).parse(ar, 0, f & Parse::TEXTMODE ?
1849                                 InsetMath::TEXT_MODE : InsetMath::MATH_MODE);
1850 }
1851
1852
1853 bool mathed_parse_cell(MathData & ar, istream & is, Parse::flags f)
1854 {
1855         return Parser(is, f).parse(ar, 0, f & Parse::TEXTMODE ?
1856                                 InsetMath::TEXT_MODE : InsetMath::MATH_MODE);
1857 }
1858
1859
1860 bool mathed_parse_normal(MathAtom & t, docstring const & str, Parse::flags f)
1861 {
1862         return Parser(str, f).parse(t);
1863 }
1864
1865
1866 bool mathed_parse_normal(MathAtom & t, Lexer & lex, Parse::flags f)
1867 {
1868         return Parser(lex, f).parse(t);
1869 }
1870
1871
1872 bool mathed_parse_normal(InsetMathGrid & grid, docstring const & str, Parse::flags f)
1873 {
1874         return Parser(str, f).parse1(grid, 0, f & Parse::TEXTMODE ?
1875                         InsetMath::TEXT_MODE : InsetMath::MATH_MODE, false);
1876 }
1877
1878
1879 void initParser()
1880 {
1881         fill(theCatcode, theCatcode + 128, catOther);
1882         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
1883         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
1884
1885         theCatcode[int('\\')] = catEscape;
1886         theCatcode[int('{')]  = catBegin;
1887         theCatcode[int('}')]  = catEnd;
1888         theCatcode[int('$')]  = catMath;
1889         theCatcode[int('&')]  = catAlign;
1890         theCatcode[int('\n')] = catNewline;
1891         theCatcode[int('#')]  = catParameter;
1892         theCatcode[int('^')]  = catSuper;
1893         theCatcode[int('_')]  = catSub;
1894         theCatcode[int(0x7f)] = catIgnore;
1895         theCatcode[int(' ')]  = catSpace;
1896         theCatcode[int('\t')] = catSpace;
1897         theCatcode[int('\r')] = catNewline;
1898         theCatcode[int('~')]  = catActive;
1899         theCatcode[int('%')]  = catComment;
1900 }
1901
1902
1903 } // namespace lyx