src/mathed/math_parser.C

   1 /**
   2  * \file math_parser.C
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  *
   8  * Full author contact details are available in file CREDITS.
   9  */
  10
  11 /*
  12
  13 If someone desperately needs partial "structures" (such as a few
  14 cells of an array inset or similar) (s)he could uses the
  15 following hack as starting point to write some macros:
  16
  17   \newif\ifcomment
  18   \commentfalse
  19   \ifcomment
  20           \def\makeamptab{\catcode`\&=4\relax}
  21           \def\makeampletter{\catcode`\&=11\relax}
  22     \def\b{\makeampletter\expandafter\makeamptab\bi}
  23     \long\def\bi#1\e{}
  24   \else
  25     \def\b{}\def\e{}
  26   \fi
  27
  28   ...
  29
  30   \[\begin{array}{ccc}
  31 1
  32 &
  33
  34   \end{array}\]
  35
  36 */
  37
  38
  39 #include <config.h>
  40
  41 #include "math_parser.h"
  42 #include "math_arrayinset.h"
  43 #include "math_braceinset.h"
  44 #include "math_charinset.h"
  45 #include "math_commentinset.h"
  46 #include "math_deliminset.h"
  47 #include "math_envinset.h"
  48 #include "math_factory.h"
  49 #include "math_kerninset.h"
  50 #include "math_macro.h"
  51 #include "math_macroarg.h"
  52 #include "math_macrotemplate.h"
  53 #include "math_parinset.h"
  54 #include "math_rootinset.h"
  55 #include "math_scriptinset.h"
  56 #include "math_sqrtinset.h"
  57 #include "math_support.h"
  58 #include "math_tabularinset.h"
  59
  60 //#include "insets/insetref.h"
  61 #include "ref_inset.h"
  62
  63 #include "lyxlex.h"
  64 #include "debug.h"
  65 #include "support/lyxlib.h"
  66
  67 #include <sstream>
  68
  69 using lyx::support::atoi;
  70
  71 using std::endl;
  72 using std::fill;
  73
  74 using std::string;
  75 using std::ios;
  76 using std::istream;
  77 using std::istringstream;
  78 using std::ostream;
  79 using std::vector;
  80
  81
  82 //#define FILEDEBUG
  83
  84
  85 namespace {
  86
  87 MathInset::mode_type asMode(MathInset::mode_type oldmode, string const & str)
  88 {
  89         //lyxerr << "handling mode: '" << str << "'" << endl;
  90         if (str == "mathmode")
  91                 return MathInset::MATH_MODE;
  92         if (str == "textmode" || str == "forcetext")
  93                 return MathInset::TEXT_MODE;
  94         return oldmode;
  95 }
  96
  97
  98 bool stared(string const & s)
  99 {
 100         string::size_type const n = s.size();
 101         return n && s[n - 1] == '*';
 102 }
 103
 104
 105 /*!
 106  * Add the row \p cellrow to \p grid.
 107  * \returns wether the row could be added. Adding a row can fail for
 108  * environments like "equation" that have a fixed number of rows.
 109  */
 110 bool addRow(MathGridInset & grid, MathGridInset::row_type & cellrow,
 111             string const & vskip)
 112 {
 113         ++cellrow;
 114         if (cellrow == grid.nrows()) {
 115                 //lyxerr << "adding row " << cellrow << endl;
 116                 grid.addRow(cellrow - 1);
 117                 if (cellrow == grid.nrows()) {
 118                         // We can't add a row to this grid, so let's
 119                         // append the content of this cell to the previous
 120                         // one.
 121                         // This does not happen in well formed .lyx files,
 122                         // but LyX versions 1.3.x and older could create
 123                         // such files and tex2lyx can still do that.
 124                         --cellrow;
 125                         lyxerr << "ignoring extra row";
 126                         if (!vskip.empty())
 127                                 lyxerr << " with extra space " << vskip;
 128                         lyxerr << '.' << endl;
 129                         return false;
 130                 }
 131         }
 132         grid.vcrskip(LyXLength(vskip), cellrow - 1);
 133         return true;
 134 }
 135
 136
 137 /*!
 138  * Add the column \p cellcol to \p grid.
 139  * \returns wether the column could be added. Adding a column can fail for
 140  * environments like "eqnarray" that have a fixed number of columns.
 141  */
 142 bool addCol(MathGridInset & grid, MathGridInset::col_type & cellcol)
 143 {
 144         ++cellcol;
 145         if (cellcol == grid.ncols()) {
 146                 //lyxerr << "adding column " << cellcol << endl;
 147                 grid.addCol(cellcol - 1);
 148                 if (cellcol == grid.ncols()) {
 149                         // We can't add a column to this grid, so let's
 150                         // append the content of this cell to the previous
 151                         // one.
 152                         // This does not happen in well formed .lyx files,
 153                         // but LyX versions 1.3.x and older could create
 154                         // such files and tex2lyx can still do that.
 155                         --cellcol;
 156                         lyxerr << "ignoring extra column." << endl;
 157                         return false;
 158                 }
 159         }
 160         return true;
 161 }
 162
 163
 164 // These are TeX's catcodes
 165 enum CatCode {
 166         catEscape,     // 0    backslash
 167         catBegin,      // 1    {
 168         catEnd,        // 2    }
 169         catMath,       // 3    $
 170         catAlign,      // 4    &
 171         catNewline,    // 5    ^^M
 172         catParameter,  // 6    #
 173         catSuper,      // 7    ^
 174         catSub,        // 8    _
 175         catIgnore,     // 9
 176         catSpace,      // 10   space
 177         catLetter,     // 11   a-zA-Z
 178         catOther,      // 12   none of the above
 179         catActive,     // 13   ~
 180         catComment,    // 14   %
 181         catInvalid     // 15   <delete>
 182 };
 183
 184 CatCode theCatcode[256];
 185
 186
 187 inline CatCode catcode(unsigned char c)
 188 {
 189         return theCatcode[c];
 190 }
 191
 192
 193 enum {
 194         FLAG_BRACE_LAST = 1 << 1,  //  last closing brace ends the parsing
 195         FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
 196         FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
 197         FLAG_BRACK_LAST = 1 << 4,  //  next closing bracket ends the parsing
 198         FLAG_TEXTMODE   = 1 << 5,  //  we are in a box
 199         FLAG_ITEM       = 1 << 6,  //  read a (possibly braced token)
 200         FLAG_LEAVE      = 1 << 7,  //  leave the loop at the end
 201         FLAG_SIMPLE     = 1 << 8,  //  next $ leaves the loop
 202         FLAG_EQUATION   = 1 << 9,  //  next \] leaves the loop
 203         FLAG_SIMPLE2    = 1 << 10, //  next \) leaves the loop
 204         FLAG_OPTION     = 1 << 11, //  read [...] style option
 205         FLAG_BRACED     = 1 << 12  //  read {...} style argument
 206 };
 207
 208
 209 //
 210 // Helper class for parsing
 211 //
 212
 213 class Token {
 214 public:
 215         ///
 216         Token() : cs_(), char_(0), cat_(catIgnore) {}
 217         ///
 218         Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
 219         ///
 220         Token(string const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
 221
 222         ///
 223         string const & cs() const { return cs_; }
 224         ///
 225         CatCode cat() const { return cat_; }
 226         ///
 227         char character() const { return char_; }
 228         ///
 229         string asString() const { return cs_.size() ? cs_ : string(1, char_); }
 230
 231 private:
 232         ///
 233         string cs_;
 234         ///
 235         char char_;
 236         ///
 237         CatCode cat_;
 238 };
 239
 240 ostream & operator<<(ostream & os, Token const & t)
 241 {
 242         if (t.cs().size())
 243                 os << '\\' << t.cs();
 244         else if (t.cat() == catLetter)
 245                 os << t.character();
 246         else
 247                 os << '[' << t.character() << ',' << t.cat() << ']';
 248         return os;
 249 }
 250
 251
 252 class Parser {
 253 public:
 254         ///
 255         typedef  MathInset::mode_type mode_type;
 256
 257         ///
 258         Parser(LyXLex & lex);
 259         ///
 260         Parser(istream & is);
 261
 262         ///
 263         bool parse(MathAtom & at);
 264         ///
 265         void parse(MathArray & array, unsigned flags, mode_type mode);
 266         ///
 267         void parse1(MathGridInset & grid, unsigned flags, mode_type mode,
 268                 bool numbered);
 269         ///
 270         MathArray parse(unsigned flags, mode_type mode);
 271         ///
 272         int lineno() const { return lineno_; }
 273         ///
 274         void putback();
 275
 276 private:
 277         ///
 278         void parse2(MathAtom & at, unsigned flags, mode_type mode, bool numbered);
 279         /// get arg delimited by 'left' and 'right'
 280         string getArg(char left, char right);
 281         ///
 282         char getChar();
 283         ///
 284         void error(string const & msg);
 285         /// dump contents to screen
 286         void dump() const;
 287         ///
 288         void tokenize(istream & is);
 289         ///
 290         void tokenize(string const & s);
 291         ///
 292         void skipSpaceTokens(istream & is, char c);
 293         ///
 294         void push_back(Token const & t);
 295         ///
 296         void pop_back();
 297         ///
 298         Token const & prevToken() const;
 299         ///
 300         Token const & nextToken() const;
 301         ///
 302         Token const & getToken();
 303         /// skips spaces if any
 304         void skipSpaces();
 305         ///
 306         void lex(string const & s);
 307         ///
 308         bool good() const;
 309         ///
 310         string parse_verbatim_item();
 311         ///
 312         string parse_verbatim_option();
 313
 314         ///
 315         int lineno_;
 316         ///
 317         vector<Token> tokens_;
 318         ///
 319         unsigned pos_;
 320 };
 321
 322
 323 Parser::Parser(LyXLex & lexer)
 324         : lineno_(lexer.getLineNo()), pos_(0)
 325 {
 326         tokenize(lexer.getStream());
 327         lexer.eatLine();
 328 }
 329
 330
 331 Parser::Parser(istream & is)
 332         : lineno_(0), pos_(0)
 333 {
 334         tokenize(is);
 335 }
 336
 337
 338 void Parser::push_back(Token const & t)
 339 {
 340         tokens_.push_back(t);
 341 }
 342
 343
 344 void Parser::pop_back()
 345 {
 346         tokens_.pop_back();
 347 }
 348
 349
 350 Token const & Parser::prevToken() const
 351 {
 352         static const Token dummy;
 353         return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
 354 }
 355
 356
 357 Token const & Parser::nextToken() const
 358 {
 359         static const Token dummy;
 360         return good() ? tokens_[pos_] : dummy;
 361 }
 362
 363
 364 Token const & Parser::getToken()
 365 {
 366         static const Token dummy;
 367         //lyxerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << endl;
 368         return good() ? tokens_[pos_++] : dummy;
 369 }
 370
 371
 372 void Parser::skipSpaces()
 373 {
 374         while (nextToken().cat() == catSpace || nextToken().cat() == catNewline)
 375                 getToken();
 376 }
 377
 378
 379 void Parser::putback()
 380 {
 381         --pos_;
 382 }
 383
 384
 385 bool Parser::good() const
 386 {
 387         return pos_ < tokens_.size();
 388 }
 389
 390
 391 char Parser::getChar()
 392 {
 393         if (!good())
 394                 error("The input stream is not well...");
 395         return tokens_[pos_++].character();
 396 }
 397
 398
 399 string Parser::getArg(char left, char right)
 400 {
 401         skipSpaces();
 402
 403         string result;
 404         char c = getChar();
 405
 406         if (c != left)
 407                 putback();
 408         else
 409                 while ((c = getChar()) != right && good())
 410                         result += c;
 411
 412         return result;
 413 }
 414
 415
 416 void Parser::skipSpaceTokens(istream & is, char c)
 417 {
 418         // skip trailing spaces
 419         while (catcode(c) == catSpace || catcode(c) == catNewline)
 420                 if (!is.get(c))
 421                         break;
 422         //lyxerr << "putting back: " << c << endl;
 423         is.putback(c);
 424 }
 425
 426
 427 void Parser::tokenize(istream & is)
 428 {
 429         // eat everything up to the next \end_inset or end of stream
 430         // and store it in s for further tokenization
 431         string s;
 432         char c;
 433         while (is.get(c)) {
 434                 s += c;
 435                 if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
 436                         s = s.substr(0, s.size() - 10);
 437                         break;
 438                 }
 439         }
 440         // Remove the space after \end_inset
 441         if (is.get(c) && c != ' ')
 442                 is.unget();
 443
 444         // tokenize buffer
 445         tokenize(s);
 446 }
 447
 448
 449 void Parser::tokenize(string const & buffer)
 450 {
 451         istringstream is(buffer, ios::in | ios::binary);
 452
 453         char c;
 454         while (is.get(c)) {
 455                 //lyxerr << "reading c: " << c << endl;
 456
 457                 switch (catcode(c)) {
 458                         case catNewline: {
 459                                 ++lineno_;
 460                                 is.get(c);
 461                                 if (catcode(c) == catNewline)
 462                                         ; //push_back(Token("par"));
 463                                 else {
 464                                         push_back(Token('\n', catNewline));
 465                                         is.putback(c);
 466                                 }
 467                                 break;
 468                         }
 469
 470 /*
 471                         case catComment: {
 472                                 while (is.get(c) && catcode(c) != catNewline)
 473                                         ;
 474                                 ++lineno_;
 475                                 break;
 476                         }
 477 */
 478
 479                         case catEscape: {
 480                                 is.get(c);
 481                                 if (!is) {
 482                                         error("unexpected end of input");
 483                                 } else {
 484                                         string s(1, c);
 485                                         if (catcode(c) == catLetter) {
 486                                                 // collect letters
 487                                                 while (is.get(c) && catcode(c) == catLetter)
 488                                                         s += c;
 489                                                 skipSpaceTokens(is, c);
 490                                         }
 491                                         push_back(Token(s));
 492                                 }
 493                                 break;
 494                         }
 495
 496                         case catSuper:
 497                         case catSub: {
 498                                 push_back(Token(c, catcode(c)));
 499                                 is.get(c);
 500                                 skipSpaceTokens(is, c);
 501                                 break;
 502                         }
 503
 504                         case catIgnore: {
 505                                 lyxerr << "ignoring a char: " << int(c) << endl;
 506                                 break;
 507                         }
 508
 509                         default:
 510                                 push_back(Token(c, catcode(c)));
 511                 }
 512         }
 513
 514 #ifdef FILEDEBUG
 515         dump();
 516 #endif
 517 }
 518
 519
 520 void Parser::dump() const
 521 {
 522         lyxerr << "\nTokens: ";
 523         for (unsigned i = 0; i < tokens_.size(); ++i) {
 524                 if (i == pos_)
 525                         lyxerr << " <#> ";
 526                 lyxerr << tokens_[i];
 527         }
 528         lyxerr << " pos: " << pos_ << endl;
 529 }
 530
 531
 532 void Parser::error(string const & msg)
 533 {
 534         lyxerr << "Line ~" << lineno_ << ": Math parse error: " << msg << endl;
 535         dump();
 536         //exit(1);
 537 }
 538
 539
 540 bool Parser::parse(MathAtom & at)
 541 {
 542         skipSpaces();
 543         MathArray ar;
 544         parse(ar, false, MathInset::UNDECIDED_MODE);
 545         if (ar.size() != 1 || ar.front()->getType() == "none") {
 546                 lyxerr << "unusual contents found: " << ar << endl;
 547                 at = MathAtom(new MathParInset(ar));
 548                 //if (at->nargs() > 0)
 549                 //      at.nucleus()->cell(0) = ar;
 550                 //else
 551                 //      lyxerr << "unusual contents found: " << ar << endl;
 552                 return true;
 553         }
 554         at = ar[0];
 555         return true;
 556 }
 557
 558
 559 string Parser::parse_verbatim_option()
 560 {
 561         skipSpaces();
 562         string res;
 563         if (nextToken().character() == '[') {
 564                 Token t = getToken();
 565                 for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
 566                         if (t.cat() == catBegin) {
 567                                 putback();
 568                                 res += '{' + parse_verbatim_item() + '}';
 569                         } else
 570                                 res += t.asString();
 571                 }
 572         }
 573         return res;
 574 }
 575
 576
 577 string Parser::parse_verbatim_item()
 578 {
 579         skipSpaces();
 580         string res;
 581         if (nextToken().cat() == catBegin) {
 582                 Token t = getToken();
 583                 for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
 584                         if (t.cat() == catBegin) {
 585                                 putback();
 586                                 res += '{' + parse_verbatim_item() + '}';
 587                         }
 588                         else
 589                                 res += t.asString();
 590                 }
 591         }
 592         return res;
 593 }
 594
 595
 596 MathArray Parser::parse(unsigned flags, mode_type mode)
 597 {
 598         MathArray ar;
 599         parse(ar, flags, mode);
 600         return ar;
 601 }
 602
 603
 604 void Parser::parse(MathArray & array, unsigned flags, mode_type mode)
 605 {
 606         MathGridInset grid(1, 1);
 607         parse1(grid, flags, mode, false);
 608         array = grid.cell(0);
 609 }
 610
 611
 612 void Parser::parse2(MathAtom & at, const unsigned flags, const mode_type mode,
 613         const bool numbered)
 614 {
 615         parse1(*(at.nucleus()->asGridInset()), flags, mode, numbered);
 616 }
 617
 618
 619 void Parser::parse1(MathGridInset & grid, unsigned flags,
 620         const mode_type mode, const bool numbered)
 621 {
 622         int limits = 0;
 623         MathGridInset::row_type cellrow = 0;
 624         MathGridInset::col_type cellcol = 0;
 625         MathArray * cell = &grid.cell(grid.index(cellrow, cellcol));
 626
 627         if (grid.asHullInset())
 628                 grid.asHullInset()->numbered(cellrow, numbered);
 629
 630         //dump();
 631         //lyxerr << " flags: " << flags << endl;
 632         //lyxerr << " mode: " << mode  << endl;
 633         //lyxerr << "grid: " << grid << endl;
 634
 635         while (good()) {
 636                 Token const & t = getToken();
 637
 638 #ifdef FILEDEBUG
 639                 lyxerr << "t: " << t << " flags: " << flags << endl;
 640                 lyxerr << "mode: " << mode  << endl;
 641                 cell->dump();
 642                 lyxerr << endl;
 643 #endif
 644
 645                 if (flags & FLAG_ITEM) {
 646
 647                         if (t.cat() == catBegin) {
 648                                 // skip the brace and collect everything to the next matching
 649                                 // closing brace
 650                                 parse1(grid, FLAG_BRACE_LAST, mode, numbered);
 651                                 return;
 652                         }
 653
 654                         // handle only this single token, leave the loop if done
 655                         flags = FLAG_LEAVE;
 656                 }
 657
 658
 659                 if (flags & FLAG_BRACED) {
 660                         if (t.cat() == catSpace)
 661                                 continue;
 662
 663                         if (t.cat() != catBegin) {
 664                                 error("opening brace expected");
 665                                 return;
 666                         }
 667
 668                         // skip the brace and collect everything to the next matching
 669                         // closing brace
 670                         flags = FLAG_BRACE_LAST;
 671                 }
 672
 673
 674                 if (flags & FLAG_OPTION) {
 675                         if (t.cat() == catOther && t.character() == '[') {
 676                                 MathArray ar;
 677                                 parse(ar, FLAG_BRACK_LAST, mode);
 678                                 cell->append(ar);
 679                         } else {
 680                                 // no option found, put back token and we are done
 681                                 putback();
 682                         }
 683                         return;
 684                 }
 685
 686                 //
 687                 // cat codes
 688                 //
 689                 if (t.cat() == catMath) {
 690                         if (mode != MathInset::MATH_MODE) {
 691                                 // we are inside some text mode thingy, so opening new math is allowed
 692                                 Token const & n = getToken();
 693                                 if (n.cat() == catMath) {
 694                                         // TeX's $$...$$ syntax for displayed math
 695                                         cell->push_back(MathAtom(new MathHullInset("equation")));
 696                                         parse2(cell->back(), FLAG_SIMPLE, MathInset::MATH_MODE, false);
 697                                         getToken(); // skip the second '$' token
 698                                 } else {
 699                                         // simple $...$  stuff
 700                                         putback();
 701                                         cell->push_back(MathAtom(new MathHullInset("simple")));
 702                                         parse2(cell->back(), FLAG_SIMPLE, MathInset::MATH_MODE, false);
 703                                 }
 704                         }
 705
 706                         else if (flags & FLAG_SIMPLE) {
 707                                 // this is the end of the formula
 708                                 return;
 709                         }
 710
 711                         else {
 712                                 error("something strange in the parser");
 713                                 break;
 714                         }
 715                 }
 716
 717                 else if (t.cat() == catLetter)
 718                         cell->push_back(MathAtom(new MathCharInset(t.character())));
 719
 720                 else if (t.cat() == catSpace && mode != MathInset::MATH_MODE) {
 721                         if (cell->empty() || cell->back()->getChar() != ' ')
 722                                 cell->push_back(MathAtom(new MathCharInset(t.character())));
 723                 }
 724
 725                 else if (t.cat() == catNewline && mode != MathInset::MATH_MODE) {
 726                         if (cell->empty() || cell->back()->getChar() != ' ')
 727                                 cell->push_back(MathAtom(new MathCharInset(' ')));
 728                 }
 729
 730                 else if (t.cat() == catParameter) {
 731                         Token const & n = getToken();
 732                         cell->push_back(MathAtom(new MathMacroArgument(n.character()-'0')));
 733                 }
 734
 735                 else if (t.cat() == catActive)
 736                         cell->push_back(MathAtom(new MathCharInset(t.character())));
 737
 738                 else if (t.cat() == catBegin) {
 739                         MathArray ar;
 740                         parse(ar, FLAG_BRACE_LAST, mode);
 741                         // do not create a BraceInset if they were written by LyX
 742                         // this helps to keep the annoyance of  "a choose b"  to a minimum
 743                         if (ar.size() == 1 && ar[0]->extraBraces())
 744                                 cell->append(ar);
 745                         else
 746                                 cell->push_back(MathAtom(new MathBraceInset(ar)));
 747                 }
 748
 749                 else if (t.cat() == catEnd) {
 750                         if (flags & FLAG_BRACE_LAST)
 751                                 return;
 752                         error("found '}' unexpectedly");
 753                         //BOOST_ASSERT(false);
 754                         //add(cell, '}', LM_TC_TEX);
 755                 }
 756
 757                 else if (t.cat() == catAlign) {
 758                         //lyxerr << " column now " << (cellcol + 1)
 759                         //       << " max: " << grid.ncols() << endl;
 760                         if (addCol(grid, cellcol))
 761                                 cell = &grid.cell(grid.index(cellrow,
 762                                                              cellcol));
 763                 }
 764
 765                 else if (t.cat() == catSuper || t.cat() == catSub) {
 766                         bool up = (t.cat() == catSuper);
 767                         // we need no new script inset if the last thing was a scriptinset,
 768                         // which has that script already not the same script already
 769                         if (!cell->size())
 770                                 cell->push_back(MathAtom(new MathScriptInset(up)));
 771                         else if (cell->back()->asScriptInset() &&
 772                                         !cell->back()->asScriptInset()->has(up))
 773                                 cell->back().nucleus()->asScriptInset()->ensure(up);
 774                         else if (cell->back()->asScriptInset())
 775                                 cell->push_back(MathAtom(new MathScriptInset(up)));
 776                         else
 777                                 cell->back() = MathAtom(new MathScriptInset(cell->back(), up));
 778                         MathScriptInset * p = cell->back().nucleus()->asScriptInset();
 779                         // special handling of {}-bases
 780                         // is this always correct?
 781                         // It appears that this is wrong (Dekel)
 782                         //if (p->nuc().size() == 1 && p->nuc().back()->asNestInset() &&
 783                         //    p->nuc().back()->extraBraces())
 784                         //      p->nuc() = p->nuc().back()->asNestInset()->cell(0);
 785                         parse(p->cell(p->idxOfScript(up)), FLAG_ITEM, mode);
 786                         if (limits) {
 787                                 p->limits(limits);
 788                                 limits = 0;
 789                         }
 790                 }
 791
 792                 else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
 793                         //lyxerr << "finished reading option" << endl;
 794                         return;
 795                 }
 796
 797                 else if (t.cat() == catOther)
 798                         cell->push_back(MathAtom(new MathCharInset(t.character())));
 799
 800                 else if (t.cat() == catComment) {
 801                         string s;
 802                         while (good()) {
 803                                 Token const & t = getToken();
 804                                 if (t.cat() == catNewline)
 805                                         break;
 806                                 s += t.asString();
 807                         }
 808                         cell->push_back(MathAtom(new MathCommentInset(s)));
 809                         skipSpaces();
 810                 }
 811
 812                 //
 813                 // control sequences
 814                 //
 815
 816                 else if (t.cs() == "lyxlock") {
 817                         if (cell->size())
 818                                 cell->back().nucleus()->lock(true);
 819                 }
 820
 821                 else if (t.cs() == "def" ||
 822                         t.cs() == "newcommand" ||
 823                         t.cs() == "renewcommand")
 824                 {
 825                         string const type = t.cs();
 826                         string name;
 827                         int nargs = 0;
 828                         if (t.cs() == "def") {
 829                                 // get name
 830                                 name = getToken().cs();
 831
 832                                 // read parameter
 833                                 string pars;
 834                                 while (good() && nextToken().cat() != catBegin) {
 835                                         pars += getToken().cs();
 836                                         ++nargs;
 837                                 }
 838                                 nargs /= 2;
 839                                 //lyxerr << "read \\def parameter list '" << pars << "'" << endl;
 840
 841                         } else { // t.cs() == "newcommand" || t.cs() == "renewcommand"
 842
 843                                 if (getToken().cat() != catBegin) {
 844                                         error("'{' in \\newcommand expected (1) ");
 845                                         return;
 846                                 }
 847
 848                                 name = getToken().cs();
 849
 850                                 if (getToken().cat() != catEnd) {
 851                                         error("'}' in \\newcommand expected");
 852                                         return;
 853                                 }
 854
 855                                 string arg  = getArg('[', ']');
 856                                 if (!arg.empty())
 857                                         nargs = atoi(arg);
 858
 859                         }
 860
 861                         MathArray ar1;
 862                         parse(ar1, FLAG_ITEM, MathInset::UNDECIDED_MODE);
 863
 864                         // we cannot handle recursive stuff at all
 865                         //MathArray test;
 866                         //test.push_back(createMathInset(name));
 867                         //if (ar1.contains(test)) {
 868                         //      error("we cannot handle recursive macros at all.");
 869                         //      return;
 870                         //}
 871
 872                         // is a version for display attached?
 873                         skipSpaces();
 874                         MathArray ar2;
 875                         if (nextToken().cat() == catBegin)
 876                                 parse(ar2, FLAG_ITEM, MathInset::MATH_MODE);
 877
 878                         cell->push_back(MathAtom(new MathMacroTemplate(name, nargs, type,
 879                                 ar1, ar2)));
 880                 }
 881
 882                 else if (t.cs() == "(") {
 883                         cell->push_back(MathAtom(new MathHullInset("simple")));
 884                         parse2(cell->back(), FLAG_SIMPLE2, MathInset::MATH_MODE, false);
 885                 }
 886
 887                 else if (t.cs() == "[") {
 888                         cell->push_back(MathAtom(new MathHullInset("equation")));
 889                         parse2(cell->back(), FLAG_EQUATION, MathInset::MATH_MODE, false);
 890                 }
 891
 892                 else if (t.cs() == "protect")
 893                         // ignore \\protect, will hopefully be re-added during output
 894                         ;
 895
 896                 else if (t.cs() == "end") {
 897                         if (flags & FLAG_END) {
 898                                 // eat environment name
 899                                 //string const name =
 900                                 getArg('{', '}');
 901                                 // FIXME: check that we ended the correct environment
 902                                 return;
 903                         }
 904                         error("found 'end' unexpectedly");
 905                 }
 906
 907                 else if (t.cs() == ")") {
 908                         if (flags & FLAG_SIMPLE2)
 909                                 return;
 910                         error("found '\\)' unexpectedly");
 911                 }
 912
 913                 else if (t.cs() == "]") {
 914                         if (flags & FLAG_EQUATION)
 915                                 return;
 916                         error("found '\\]' unexpectedly");
 917                 }
 918
 919                 else if (t.cs() == "\\") {
 920                         if (addRow(grid, cellrow, getArg('[', ']'))) {
 921                                 cellcol = 0;
 922                                 if (grid.asHullInset())
 923                                         grid.asHullInset()->numbered(
 924                                                         cellrow, numbered);
 925                                 cell = &grid.cell(grid.index(cellrow,
 926                                                              cellcol));
 927                         }
 928                 }
 929
 930 #if 0
 931                 else if (t.cs() == "multicolumn") {
 932                         // extract column count and insert dummy cells
 933                         MathArray count;
 934                         parse(count, FLAG_ITEM, mode);
 935                         int cols = 1;
 936                         if (!extractNumber(count, cols)) {
 937                                 lyxerr << " can't extract number of cells from " << count << endl;
 938                         }
 939                         // resize the table if necessary
 940                         for (int i = 0; i < cols; ++i) {
 941                                 if (addCol(grid, cellcol)) {
 942                                         cell = &grid.cell(grid.index(
 943                                                         cellrow, cellcol));
 944                                         // mark this as dummy
 945                                         grid.cellinfo(grid.index(
 946                                                 cellrow, cellcol)).dummy_ = true;
 947                                 }
 948                         }
 949                         // the last cell is the real thing, not a dummy
 950                         grid.cellinfo(grid.index(cellrow, cellcol)).dummy_ = false;
 951
 952                         // read special alignment
 953                         MathArray align;
 954                         parse(align, FLAG_ITEM, mode);
 955                         //grid.cellinfo(grid.index(cellrow, cellcol)).align_ = extractString(align);
 956
 957                         // parse the remaining contents into the "real" cell
 958                         parse(*cell, FLAG_ITEM, mode);
 959                 }
 960 #endif
 961
 962                 else if (t.cs() == "limits")
 963                         limits = 1;
 964
 965                 else if (t.cs() == "nolimits")
 966                         limits = -1;
 967
 968                 else if (t.cs() == "nonumber") {
 969                         if (grid.asHullInset())
 970                                 grid.asHullInset()->numbered(cellrow, false);
 971                 }
 972
 973                 else if (t.cs() == "number") {
 974                         if (grid.asHullInset())
 975                                 grid.asHullInset()->numbered(cellrow, true);
 976                 }
 977
 978                 else if (t.cs() == "hline") {
 979                         grid.rowinfo(cellrow).lines_ ++;
 980                 }
 981
 982                 else if (t.cs() == "sqrt") {
 983                         MathArray ar;
 984                         parse(ar, FLAG_OPTION, mode);
 985                         if (ar.size()) {
 986                                 cell->push_back(MathAtom(new MathRootInset));
 987                                 cell->back().nucleus()->cell(0) = ar;
 988                                 parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
 989                         } else {
 990                                 cell->push_back(MathAtom(new MathSqrtInset));
 991                                 parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
 992                         }
 993                 }
 994
 995                 else if (t.cs() == "xrightarrow" || t.cs() == "xleftarrow") {
 996                         cell->push_back(createMathInset(t.cs()));
 997                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, mode);
 998                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
 999                 }
1000
1001                 else if (t.cs() == "ref" || t.cs() == "prettyref" ||
1002                                 t.cs() == "pageref" || t.cs() == "vpageref" || t.cs() == "vref") {
1003                         cell->push_back(MathAtom(new RefInset(t.cs())));
1004                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, mode);
1005                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1006                 }
1007
1008                 else if (t.cs() == "left") {
1009                         skipSpaces();
1010                         string l = getToken().asString();
1011                         MathArray ar;
1012                         parse(ar, FLAG_RIGHT, mode);
1013                         skipSpaces();
1014                         string r = getToken().asString();
1015                         cell->push_back(MathAtom(new MathDelimInset(l, r, ar)));
1016                 }
1017
1018                 else if (t.cs() == "right") {
1019                         if (flags & FLAG_RIGHT)
1020                                 return;
1021                         //lyxerr << "got so far: '" << cell << "'" << endl;
1022                         error("Unmatched right delimiter");
1023                         return;
1024                 }
1025
1026                 else if (t.cs() == "begin") {
1027                         string const name = getArg('{', '}');
1028
1029                         if (name == "array" || name == "subarray") {
1030                                 string const valign = parse_verbatim_option() + 'c';
1031                                 string const halign = parse_verbatim_item();
1032                                 cell->push_back(MathAtom(new MathArrayInset(name, valign[0], halign)));
1033                                 parse2(cell->back(), FLAG_END, mode, false);
1034                         }
1035
1036                         else if (name == "tabular") {
1037                                 string const valign = parse_verbatim_option() + 'c';
1038                                 string const halign = parse_verbatim_item();
1039                                 cell->push_back(MathAtom(new MathTabularInset(name, valign[0], halign)));
1040                                 parse2(cell->back(), FLAG_END, MathInset::TEXT_MODE, false);
1041                         }
1042
1043                         else if (name == "split" || name == "cases" ||
1044                                          name == "gathered" || name == "aligned" ||
1045                                    name == "alignedat") {
1046                                 cell->push_back(createMathInset(name));
1047                                 parse2(cell->back(), FLAG_END, mode, false);
1048                         }
1049
1050                         else if (name == "math") {
1051                                 cell->push_back(MathAtom(new MathHullInset("simple")));
1052                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, true);
1053                         }
1054
1055                         else if (name == "equation" || name == "equation*"
1056                                         || name == "displaymath") {
1057                                 cell->push_back(MathAtom(new MathHullInset("equation")));
1058                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, (name == "equation"));
1059                         }
1060
1061                         else if (name == "eqnarray" || name == "eqnarray*") {
1062                                 cell->push_back(MathAtom(new MathHullInset("eqnarray")));
1063                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1064                         }
1065
1066                         else if (name == "align" || name == "align*") {
1067                                 cell->push_back(MathAtom(new MathHullInset("align")));
1068                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1069                         }
1070
1071                         else if (name == "flalign" || name == "flalign*") {
1072                                 cell->push_back(MathAtom(new MathHullInset("flalign")));
1073                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1074                         }
1075
1076                         else if (name == "alignat" || name == "alignat*") {
1077                                 // ignore this for a while
1078                                 getArg('{', '}');
1079                                 cell->push_back(MathAtom(new MathHullInset("alignat")));
1080                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1081                         }
1082
1083                         else if (name == "xalignat" || name == "xalignat*") {
1084                                 // ignore this for a while
1085                                 getArg('{', '}');
1086                                 cell->push_back(MathAtom(new MathHullInset("xalignat")));
1087                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1088                         }
1089
1090                         else if (name == "xxalignat") {
1091                                 // ignore this for a while
1092                                 getArg('{', '}');
1093                                 cell->push_back(MathAtom(new MathHullInset("xxalignat")));
1094                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1095                         }
1096
1097                         else if (name == "multline" || name == "multline*") {
1098                                 cell->push_back(MathAtom(new MathHullInset("multline")));
1099                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1100                         }
1101
1102                         else if (name == "gather" || name == "gather*") {
1103                                 cell->push_back(MathAtom(new MathHullInset("gather")));
1104                                 parse2(cell->back(), FLAG_END, MathInset::MATH_MODE, !stared(name));
1105                         }
1106
1107                         else if (latexkeys const * l = in_word_set(name)) {
1108                                 if (l->inset == "matrix") {
1109                                         cell->push_back(createMathInset(name));
1110                                         parse2(cell->back(), FLAG_END, mode, false);
1111                                 }
1112                         }
1113
1114                         else {
1115                                 dump();
1116                                 lyxerr << "found unknown math environment '" << name << "'" << endl;
1117                                 // create generic environment inset
1118                                 cell->push_back(MathAtom(new MathEnvInset(name)));
1119                                 parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
1120                         }
1121                 }
1122
1123                 else if (t.cs() == "kern") {
1124 #ifdef WITH_WARNINGS
1125 #warning A hack...
1126 #endif
1127                         string s;
1128                         while (true) {
1129                                 Token const & t = getToken();
1130                                 if (!good()) {
1131                                         putback();
1132                                         break;
1133                                 }
1134                                 s += t.character();
1135                                 if (isValidLength(s))
1136                                         break;
1137                         }
1138                         cell->push_back(MathAtom(new MathKernInset(s)));
1139                 }
1140
1141                 else if (t.cs() == "label") {
1142                         string label = parse_verbatim_item();
1143                         MathArray ar;
1144                         asArray(label, ar);
1145                         if (grid.asHullInset()) {
1146                                 grid.asHullInset()->label(cellrow, label);
1147                         } else {
1148                                 cell->push_back(createMathInset(t.cs()));
1149                                 cell->push_back(MathAtom(new MathBraceInset(ar)));
1150                         }
1151                 }
1152
1153                 else if (t.cs() == "choose" || t.cs() == "over" || t.cs() == "atop") {
1154                         MathAtom at = createMathInset(t.cs());
1155                         at.nucleus()->cell(0) = *cell;
1156                         cell->clear();
1157                         parse(at.nucleus()->cell(1), flags, mode);
1158                         cell->push_back(at);
1159                         return;
1160                 }
1161
1162                 else if (t.cs() == "color") {
1163                         MathAtom at = createMathInset(t.cs());
1164                         parse(at.nucleus()->cell(0), FLAG_ITEM, MathInset::TEXT_MODE);
1165                         parse(at.nucleus()->cell(1), flags, mode);
1166                         cell->push_back(at);
1167                         return;
1168                 }
1169
1170                 else if (t.cs() == "substack") {
1171                         cell->push_back(createMathInset(t.cs()));
1172                         parse2(cell->back(), FLAG_ITEM, mode, false);
1173                 }
1174
1175                 else if (t.cs() == "framebox" || t.cs() == "makebox") {
1176                         cell->push_back(createMathInset(t.cs()));
1177                         parse(cell->back().nucleus()->cell(0), FLAG_OPTION, MathInset::TEXT_MODE);
1178                         parse(cell->back().nucleus()->cell(1), FLAG_OPTION, MathInset::TEXT_MODE);
1179                         parse(cell->back().nucleus()->cell(2), FLAG_ITEM, MathInset::TEXT_MODE);
1180                 }
1181
1182 #if 0
1183                 else if (t.cs() == "infer") {
1184                         MathArray ar;
1185                         parse(ar, FLAG_OPTION, mode);
1186                         cell->push_back(createMathInset(t.cs()));
1187                         parse2(cell->back(), FLAG_ITEM, mode, false);
1188                 }
1189
1190                 // Disabled
1191                 else if (1 && t.cs() == "ar") {
1192                         auto_ptr<MathXYArrowInset> p(new MathXYArrowInset);
1193                         // try to read target
1194                         parse(p->cell(0), FLAG_OTPTION, mode);
1195                         // try to read label
1196                         if (nextToken().cat() == catSuper || nextToken().cat() == catSub) {
1197                                 p->up_ = nextToken().cat() == catSuper;
1198                                 getToken();
1199                                 parse(p->cell(1), FLAG_ITEM, mode);
1200                                 //lyxerr << "read label: " << p->cell(1) << endl;
1201                         }
1202
1203                         cell->push_back(MathAtom(p.release()));
1204                         //lyxerr << "read cell: " << cell << endl;
1205                 }
1206 #endif
1207
1208                 else if (t.cs().size()) {
1209                         latexkeys const * l = in_word_set(t.cs());
1210                         if (l) {
1211                                 if (l->inset == "font") {
1212                                         cell->push_back(createMathInset(t.cs()));
1213                                         parse(cell->back().nucleus()->cell(0), FLAG_ITEM, asMode(mode, l->extra));
1214                                 }
1215
1216                                 else if (l->inset == "oldfont") {
1217                                         cell->push_back(createMathInset(t.cs()));
1218                                         parse(cell->back().nucleus()->cell(0), flags, asMode(mode, l->extra));
1219                                         return;
1220                                 }
1221
1222                                 else if (l->inset == "style") {
1223                                         cell->push_back(createMathInset(t.cs()));
1224                                         parse(cell->back().nucleus()->cell(0), flags, mode);
1225                                         return;
1226                                 }
1227
1228                                 else {
1229                                         MathAtom at = createMathInset(t.cs());
1230                                         for (MathInset::idx_type i = 0; i < at->nargs(); ++i)
1231                                                 parse(at.nucleus()->cell(i), FLAG_ITEM, asMode(mode, l->extra));
1232                                         cell->push_back(at);
1233                                 }
1234                         }
1235
1236                         else {
1237                                 MathAtom at = createMathInset(t.cs());
1238                                 MathInset::mode_type m = mode;
1239                                 //if (m == MathInset::UNDECIDED_MODE)
1240                                 //lyxerr << "default creation: m1: " << m << endl;
1241                                 if (at->currentMode() != MathInset::UNDECIDED_MODE)
1242                                         m = at->currentMode();
1243                                 //lyxerr << "default creation: m2: " << m << endl;
1244                                 MathInset::idx_type start = 0;
1245                                 // this fails on \bigg[...\bigg]
1246                                 //MathArray opt;
1247                                 //parse(opt, FLAG_OPTION, MathInset::VERBATIM_MODE);
1248                                 //if (opt.size()) {
1249                                 //      start = 1;
1250                                 //      at.nucleus()->cell(0) = opt;
1251                                 //}
1252                                 for (MathInset::idx_type i = start; i < at->nargs(); ++i) {
1253                                         parse(at.nucleus()->cell(i), FLAG_ITEM, m);
1254                                         skipSpaces();
1255                                 }
1256                                 cell->push_back(at);
1257                         }
1258                 }
1259
1260
1261                 if (flags & FLAG_LEAVE) {
1262                         flags &= ~FLAG_LEAVE;
1263                         break;
1264                 }
1265         }
1266 }
1267
1268
1269
1270 } // anonymous namespace
1271
1272
1273 void mathed_parse_cell(MathArray & ar, string const & str)
1274 {
1275         istringstream is(str);
1276         mathed_parse_cell(ar, is);
1277 }
1278
1279
1280 void mathed_parse_cell(MathArray & ar, istream & is)
1281 {
1282         Parser(is).parse(ar, 0, MathInset::MATH_MODE);
1283 }
1284
1285
1286 bool mathed_parse_normal(MathAtom & t, string const & str)
1287 {
1288         istringstream is(str);
1289         return Parser(is).parse(t);
1290 }
1291
1292
1293 bool mathed_parse_normal(MathAtom & t, istream & is)
1294 {
1295         return Parser(is).parse(t);
1296 }
1297
1298
1299 bool mathed_parse_normal(MathAtom & t, LyXLex & lex)
1300 {
1301         return Parser(lex).parse(t);
1302 }
1303
1304
1305 void mathed_parse_normal(MathGridInset & grid, string const & str)
1306 {
1307         istringstream is(str);
1308         Parser(is).parse1(grid, 0, MathInset::MATH_MODE, false);
1309 }
1310
1311
1312 void initParser()
1313 {
1314         fill(theCatcode, theCatcode + 256, catOther);
1315         fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
1316         fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
1317
1318         theCatcode[int('\\')] = catEscape;
1319         theCatcode[int('{')]  = catBegin;
1320         theCatcode[int('}')]  = catEnd;
1321         theCatcode[int('$')]  = catMath;
1322         theCatcode[int('&')]  = catAlign;
1323         theCatcode[int('\n')] = catNewline;
1324         theCatcode[int('#')]  = catParameter;
1325         theCatcode[int('^')]  = catSuper;
1326         theCatcode[int('_')]  = catSub;
1327         theCatcode[int(0x7f)] = catIgnore;
1328         theCatcode[int(' ')]  = catSpace;
1329         theCatcode[int('\t')] = catSpace;
1330         theCatcode[int('\r')] = catNewline;
1331         theCatcode[int('~')]  = catActive;
1332         theCatcode[int('%')]  = catComment;
1333 }