]> git.lyx.org Git - lyx.git/blobdiff - src/mathed/MathParser.cpp
MathML: remove redundant mrow in InsetMathScript.
[lyx.git] / src / mathed / MathParser.cpp
index de2d4b691af996ee0b4295c836c1d181b632df62..6d1b74741e74540d2aa7ce040b7b8aa13771eb1f 100644 (file)
@@ -51,7 +51,7 @@ following hack as starting point to write some macros:
 #include "InsetMathEnv.h"
 #include "InsetMathFrac.h"
 #include "InsetMathKern.h"
-#include "MathMacro.h"
+#include "InsetMathMacro.h"
 #include "InsetMathPar.h"
 #include "InsetMathRef.h"
 #include "InsetMathRoot.h"
@@ -63,9 +63,10 @@ following hack as starting point to write some macros:
 #include "InsetMathStackrel.h"
 #include "InsetMathString.h"
 #include "InsetMathTabular.h"
-#include "MathMacroTemplate.h"
+#include "InsetMathMacroTemplate.h"
+#include "MathExtern.h"
 #include "MathFactory.h"
-#include "MathMacroArgument.h"
+#include "InsetMathMacroArgument.h"
 #include "MathSupport.h"
 
 #include "Buffer.h"
@@ -73,8 +74,8 @@ following hack as starting point to write some macros:
 #include "Encoding.h"
 #include "Lexer.h"
 
-#include "support/debug.h"
 #include "support/convert.h"
+#include "support/debug.h"
 #include "support/docstream.h"
 
 #include <sstream>
@@ -156,11 +157,11 @@ docstring escapeSpecialChars(docstring const & str, bool textmode)
 
 /*!
  * Add the row \p cellrow to \p grid.
- * \returns wether the row could be added. Adding a row can fail for
+ * \returns whether the row could be added. Adding a row can fail for
  * environments like "equation" that have a fixed number of rows.
  */
-bool addRow(InsetMathGrid & grid, InsetMathGrid::row_type & cellrow,
-           docstring const & vskip, bool allow_newpage_ = true)
+bool addRow(InsetMathGrid & grid, row_type & cellrow,
+           docstring const & vskip, bool allow_newpage = true)
 {
        ++cellrow;
        if (cellrow == grid.nrows()) {
@@ -177,24 +178,24 @@ bool addRow(InsetMathGrid & grid, InsetMathGrid::row_type & cellrow,
                        lyxerr << "ignoring extra row";
                        if (!vskip.empty())
                                lyxerr << " with extra space " << to_utf8(vskip);
-                       if (!allow_newpage_)
+                       if (!allow_newpage)
                                lyxerr << " with no page break allowed";
                        lyxerr << '.' << endl;
                        return false;
                }
        }
        grid.vcrskip(Length(to_utf8(vskip)), cellrow - 1);
-       grid.rowinfo(cellrow - 1).allow_newpage_ = allow_newpage_;
+       grid.rowinfo(cellrow - 1).allow_newpage = allow_newpage;
        return true;
 }
 
 
 /*!
  * Add the column \p cellcol to \p grid.
- * \returns wether the column could be added. Adding a column can fail for
+ * \returns whether the column could be added. Adding a column can fail for
  * environments like "eqnarray" that have a fixed number of columns.
  */
-bool addCol(InsetMathGrid & grid, InsetMathGrid::col_type & cellcol)
+bool addCol(InsetMathGrid & grid, col_type & cellcol)
 {
        ++cellcol;
        if (cellcol == grid.ncols()) {
@@ -228,14 +229,18 @@ bool addCol(InsetMathGrid & grid, InsetMathGrid::col_type & cellcol)
  * \endverbatim
  * will result in a grid with 3 rows (+ the dummy row that is always present),
  * because the last '\\' opens a new row.
+ * Do never delete a row that contains a multicolumn, even if all cells empty,
+ * since the multicolumn information would get lost otherwise.
  * Note that this is only needed for inner-hull grid types, such as array
  * or aligned, but not for outer-hull grid types, such as eqnarray or align.
  */
 void delEmptyLastRow(InsetMathGrid & grid)
 {
-       InsetMathGrid::row_type const row = grid.nrows() - 1;
-       for (InsetMathGrid::col_type col = 0; col < grid.ncols(); ++col) {
-               if (!grid.cell(grid.index(row, col)).empty())
+       row_type const row = grid.nrows() - 1;
+       for (col_type col = 0; col < grid.ncols(); ++col) {
+               idx_type const idx = grid.index(row, col);
+               if (!grid.cell(idx).empty() ||
+                   grid.cellinfo(idx).multi != InsetMathGrid::CELL_NORMAL)
                        return;
        }
        // Copy the row information of the empty row (which would contain the
@@ -354,7 +359,7 @@ ostream & operator<<(ostream & os, Token const & t)
 {
        if (!t.cs().empty()) {
                docstring const & cs = t.cs();
-               // FIXME: For some strange reason, the stream operator instanciate
+               // FIXME: For some strange reason, the stream operator instantiate
                // a new Token before outputting the contents of t.cs().
                // Because of this the line
                //     os << '\\' << cs;
@@ -365,9 +370,9 @@ ostream & operator<<(ostream & os, Token const & t)
                os << '\\' << to_utf8(cs);
        }
        else if (t.cat() == catLetter)
-               os << t.character();
+               os << static_cast<uint32_t>(t.character());
        else
-               os << '[' << t.character() << ',' << t.cat() << ']';
+               os << '[' << static_cast<uint32_t>(t.character()) << ',' << t.cat() << ']';
        return os;
 }
 
@@ -395,11 +400,15 @@ public:
        bool parse1(InsetMathGrid & grid, unsigned flags, mode_type mode,
                bool numbered);
        ///
-       MathData parse(unsigned flags, mode_type mode);
-       ///
        int lineno() const { return lineno_; }
        ///
        void putback();
+       /// store current position
+       void pushPosition();
+       /// restore previous position
+       void popPosition();
+       /// forget last saved position
+       void dropPosition();
 
 private:
        ///
@@ -423,8 +432,6 @@ private:
        ///
        void push_back(Token const & t);
        ///
-       void pop_back();
-       ///
        Token const & prevToken() const;
        ///
        Token const & nextToken() const;
@@ -447,6 +454,8 @@ private:
        vector<Token> tokens_;
        ///
        unsigned pos_;
+       ///
+       std::vector<unsigned> positions_;
        /// Stack of active environments
        vector<docstring> environments_;
        ///
@@ -487,12 +496,6 @@ void Parser::push_back(Token const & t)
 }
 
 
-void Parser::pop_back()
-{
-       tokens_.pop_back();
-}
-
-
 Token const & Parser::prevToken() const
 {
        static const Token dummy;
@@ -528,6 +531,25 @@ void Parser::putback()
 }
 
 
+void Parser::pushPosition()
+{
+       positions_.push_back(pos_);
+}
+
+
+void Parser::popPosition()
+{
+       pos_ = positions_.back();
+       positions_.pop_back();
+}
+
+
+void Parser::dropPosition()
+{
+       positions_.pop_back();
+}
+
+
 bool Parser::good() const
 {
        return pos_ < tokens_.size();
@@ -599,7 +621,7 @@ void Parser::tokenize(istream & is)
 
 void Parser::tokenize(docstring const & buffer)
 {
-       idocstringstream is(mode_ & Parse::VERBATIM
+       idocstringstream is((mode_ & Parse::VERBATIM)
                        ? escapeSpecialChars(buffer, mode_ & Parse::TEXTMODE)
                        : buffer, ios::in | ios::binary);
 
@@ -721,7 +743,8 @@ docstring Parser::parse_verbatim_option()
        skipSpaces();
        docstring res;
        if (nextToken().character() == '[') {
-               Token t = getToken();
+               // eat [
+               getToken();
                for (Token t = getToken(); t.character() != ']' && good(); t = getToken()) {
                        if (t.cat() == catBegin) {
                                putback();
@@ -739,7 +762,8 @@ docstring Parser::parse_verbatim_item()
        skipSpaces();
        docstring res;
        if (nextToken().cat() == catBegin) {
-               Token t = getToken();
+               // eat catBegin
+               getToken();
                for (Token t = getToken(); t.cat() != catEnd && good(); t = getToken()) {
                        if (t.cat() == catBegin) {
                                putback();
@@ -753,14 +777,6 @@ docstring Parser::parse_verbatim_item()
 }
 
 
-MathData Parser::parse(unsigned flags, mode_type mode)
-{
-       MathData ar(buffer_);
-       parse(ar, flags, mode);
-       return ar;
-}
-
-
 bool Parser::parse(MathData & array, unsigned flags, mode_type mode)
 {
        InsetMathGrid grid(buffer_, 1, 1);
@@ -780,9 +796,8 @@ void Parser::parse2(MathAtom & at, const unsigned flags, const mode_type mode,
 bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
        const mode_type mode, const bool numbered)
 {
-       int limits = 0;
-       InsetMathGrid::row_type cellrow = 0;
-       InsetMathGrid::col_type cellcol = 0;
+       row_type cellrow = 0;
+       col_type cellcol = 0;
        MathData * cell = &grid.cell(grid.index(cellrow, cellcol));
        Buffer * buf = buffer_;
 
@@ -914,8 +929,13 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                }
 
                else if (t.cat() == catParameter) {
-                       Token const & n = getToken();
-                       cell->push_back(MathAtom(new MathMacroArgument(n.character()-'0')));
+                       Token const & n = nextToken();
+                       char_type c = n.character();
+                       if (c && '0' < c && c <= '9') {
+                               cell->push_back(MathAtom(new InsetMathMacroArgument(c - '0')));
+                               getToken();
+                       } else
+                               cell->push_back(MathAtom(new InsetMathHash()));
                }
 
                else if (t.cat() == catActive)
@@ -984,10 +1004,6 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                p->nuc().erase(0);
 
                        parse(p->cell(p->idxOfScript(up)), FLAG_ITEM, mode);
-                       if (limits) {
-                               p->limits(limits);
-                               limits = 0;
-                       }
                }
 
                else if (t.character() == ']' && (flags & FLAG_BRACK_LAST)) {
@@ -997,7 +1013,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
 
                else if (t.cat() == catOther) {
                        char_type c = t.character();
-                       if (isAsciiOrMathAlpha(c)
+                       if (!Encodings::isUnicodeTextOnly(c)
                            || mode_ & Parse::VERBATIM
                            || !(mode_ & Parse::USETEXT)
                            || mode == InsetMath::TEXT_MODE) {
@@ -1006,7 +1022,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                MathAtom at = createInsetMath("text", buf);
                                at.nucleus()->cell(0).push_back(MathAtom(new InsetMathChar(c)));
                                while (nextToken().cat() == catOther
-                                      && !isAsciiOrMathAlpha(nextToken().character())) {
+                                      && Encodings::isUnicodeTextOnly(nextToken().character())) {
                                        c = getToken().character();
                                        at.nucleus()->cell(0).push_back(MathAtom(new InsetMathChar(c)));
                                }
@@ -1017,10 +1033,10 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                else if (t.cat() == catComment) {
                        docstring s;
                        while (good()) {
-                               Token const & t = getToken();
-                               if (t.cat() == catNewline)
+                               Token const & tt = getToken();
+                               if (tt.cat() == catNewline)
                                        break;
-                               s += t.asInput();
+                               s += tt.asInput();
                        }
                        cell->push_back(MathAtom(new InsetMathComment(buf, s)));
                        skipSpaces();
@@ -1039,10 +1055,10 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                         t.cs() == "def") {
                        if (t.cs() == "global")
                                getToken();
-                       
+
                        // get name
                        docstring name = getToken().cs();
-                       
+
                        // read parameters
                        int nargs = 0;
                        docstring pars;
@@ -1051,25 +1067,25 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                ++nargs;
                        }
                        nargs /= 2;
-                       
+
                        // read definition
                        MathData def;
                        parse(def, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
-                       
+
                        // is a version for display attached?
                        skipSpaces();
                        MathData display;
                        if (nextToken().cat() == catBegin)
                                parse(display, FLAG_ITEM, InsetMath::MATH_MODE);
-                       
-                       cell->push_back(MathAtom(new MathMacroTemplate(buf,
+
+                       cell->push_back(MathAtom(new InsetMathMacroTemplate(buf,
                                name, nargs, 0, MacroTypeDef,
                                vector<MathData>(), def, display)));
 
                        if (buf && (mode_ & Parse::TRACKMACRO))
                                buf->usermacros.insert(name);
                }
-               
+
                else if (t.cs() == "newcommand" ||
                         t.cs() == "renewcommand" ||
                         t.cs() == "newlyxcommand") {
@@ -1083,13 +1099,13 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                error("'}' in \\newcommand expected");
                                return success_;
                        }
-                               
+
                        // get arity
                        docstring const arg = getArg('[', ']');
                        int nargs = 0;
                        if (!arg.empty())
                                nargs = convert<int>(arg);
-                               
+
                        // optional argument given?
                        skipSpaces();
                        int optionals = 0;
@@ -1100,24 +1116,24 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                parse(optionalValues[optionals], FLAG_BRACK_LAST, mode);
                                ++optionals;
                        }
-                       
+
                        MathData def;
                        parse(def, FLAG_ITEM, InsetMath::UNDECIDED_MODE);
-                       
+
                        // is a version for display attached?
                        skipSpaces();
                        MathData display;
                        if (nextToken().cat() == catBegin)
                                parse(display, FLAG_ITEM, InsetMath::MATH_MODE);
-                       
-                       cell->push_back(MathAtom(new MathMacroTemplate(buf,
+
+                       cell->push_back(MathAtom(new InsetMathMacroTemplate(buf,
                                name, nargs, optionals, MacroTypeNewcommand,
                                optionalValues, def, display)));
 
                        if (buf && (mode_ & Parse::TRACKMACRO))
                                buf->usermacros.insert(name);
                }
-               
+
                else if (t.cs() == "newcommandx" ||
                         t.cs() == "renewcommandx") {
                        // \newcommandx{\foo}[2][usedefault, addprefix=\global,1=default]{#1,#2}
@@ -1132,7 +1148,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                }
                        } else
                                name = getToken().cs();
-                               
+
                        // get arity
                        docstring const arg = getArg('[', ']');
                        if (arg.empty()) {
@@ -1140,14 +1156,14 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                return success_;
                        }
                        int nargs = convert<int>(arg);
-                       
+
                        // get options
                        int optionals = 0;
                        vector<MathData> optionalValues;
                        if (nextToken().character() == '[') {
                                // skip '['
                                getToken();
-                                       
+
                                // handle 'opt=value' options, separated by ','.
                                skipSpaces();
                                while (nextToken().character() != ']' && good()) {
@@ -1160,14 +1176,14 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                                              "for given optional parameter.");
                                                        return success_;
                                                }
-                                               
+
                                                // skip '='
                                                if (getToken().character() != '=') {
                                                        error("'=' and optional parameter value "
                                                              "expected for \\newcommandx");
                                                        return success_;
                                                }
-                                               
+
                                                // get value
                                                int optNum = max(size_t(n), optionalValues.size());
                                                optionalValues.resize(optNum);
@@ -1182,12 +1198,12 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                        } else if (nextToken().cat() == catLetter) {
                                                // we in fact ignore every non-optional
                                                // parameter
-                                               
+
                                                // get option name
                                                docstring opt;
                                                while (nextToken().cat() == catLetter)
                                                        opt += getChar();
-                                       
+
                                                // value?
                                                skipSpaces();
                                                MathData value;
@@ -1195,14 +1211,14 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                                        getToken();
                                                        while (nextToken().character() != ']'
                                                                && nextToken().character() != ',')
-                                                               parse(value, FLAG_ITEM, 
+                                                               parse(value, FLAG_ITEM,
                                                                      InsetMath::UNDECIDED_MODE);
                                                }
                                        } else {
                                                error("option for \\newcommandx expected");
                                                return success_;
                                        }
-                                       
+
                                        // skip komma
                                        skipSpaces();
                                        if (nextToken().character() == ',') {
@@ -1214,7 +1230,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                                return success_;
                                        }
                                }
-                               
+
                                // skip ']'
                                if (!good())
                                        return success_;
@@ -1231,7 +1247,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        if (nextToken().cat() == catBegin)
                                parse(display, FLAG_ITEM, InsetMath::MATH_MODE);
 
-                       cell->push_back(MathAtom(new MathMacroTemplate(buf,
+                       cell->push_back(MathAtom(new InsetMathMacroTemplate(buf,
                                name, nargs, optionals, MacroTypeNewcommandx,
                                optionalValues, def, display)));
 
@@ -1307,14 +1323,36 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                else if (t.cs() == "\\") {
                        if (flags & FLAG_ALIGN)
                                return success_;
-                       bool added = false;
+                       bool starred = false;
+                       docstring arg;
                        if (nextToken().asInput() == "*") {
                                getToken();
-                               added = addRow(grid, cellrow, docstring(), false);
-                       } else if (good())
-                               added = addRow(grid, cellrow, getArg('[', ']'));
-                       else
+                               starred = true;
+                       } else if (nextToken().asInput() == "[")
+                               arg = getArg('[', ']');
+                       else if (!good())
                                error("missing token after \\\\");
+                       // skip "{}" added in front of "[" (the
+                       // counterpart is in InsetMathGrid::eolString())
+                       // skip spaces because formula could come from tex2lyx
+                       bool skipBraces = false;
+                       pushPosition();
+                       if (nextToken().cat() == catBegin) {
+                               getToken();
+                               if (nextToken().cat() == catEnd) {
+                                       getToken();
+                                       pushPosition();
+                                       skipSpaces();
+                                       if (nextToken().asInput() == "[")
+                                               skipBraces = true;
+                                       popPosition();
+                               }
+                       }
+                       if (skipBraces)
+                               dropPosition();
+                       else
+                               popPosition();
+                       bool const added = addRow(grid, cellrow, arg, !starred);
                        if (added) {
                                cellcol = 0;
                                if (grid.asHullInset())
@@ -1325,43 +1363,47 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        }
                }
 
-#if 0
-               else if (t.cs() == "multicolumn") {
-                       // extract column count and insert dummy cells
+               else if (t.cs() == "multicolumn" && grid.handlesMulticolumn()) {
+                       // if the columns are specified numerically,
+                       // extract column count and insert dummy cells,
+                       // otherwise parse it as an user macro
                        MathData count;
                        parse(count, FLAG_ITEM, mode);
-                       int cols = 1;
-                       if (!extractNumber(count, cols)) {
-                               success_ = false;
-                               lyxerr << " can't extract number of cells from " << count << endl;
-                       }
-                       // resize the table if necessary
-                       for (int i = 0; i < cols; ++i) {
-                               if (addCol(grid, cellcol)) {
-                                       cell = &grid.cell(grid.index(
-                                                       cellrow, cellcol));
-                                       // mark this as dummy
-                                       grid.cellinfo(grid.index(
-                                               cellrow, cellcol)).dummy_ = true;
+                       int cols;
+                       // limit arbitrarily to 100 columns
+                       if (extractNumber(count, cols) && cols > 0 && cols < 100) {
+                               // resize the table if necessary
+                               size_t first = grid.index(cellrow, cellcol);
+                               for (int i = 1; i < cols; ++i) {
+                                       if (addCol(grid, cellcol)) {
+                                               size_t const idx = grid.index(cellrow, cellcol);
+                                               grid.cellinfo(idx).multi =
+                                                       InsetMathGrid::CELL_PART_OF_MULTICOLUMN;
+                                       }
                                }
-                       }
-                       // the last cell is the real thing, not a dummy
-                       grid.cellinfo(grid.index(cellrow, cellcol)).dummy_ = false;
 
-                       // read special alignment
-                       MathData align;
-                       parse(align, FLAG_ITEM, mode);
-                       //grid.cellinfo(grid.index(cellrow, cellcol)).align_ = extractString(align);
+                               // the first cell is the real thing, not a dummy
+                               cell = &grid.cell(first);
+                               grid.cellinfo(first).multi =
+                                       InsetMathGrid::CELL_BEGIN_OF_MULTICOLUMN;
+
+                               // read special alignment
+                               MathData align;
+                               parse(align, FLAG_ITEM, mode);
+                               grid.cellinfo(first).align = asString(align);
 
-                       // parse the remaining contents into the "real" cell
-                       parse(*cell, FLAG_ITEM, mode);
+                               // parse the remaining contents into the "real" cell
+                               parse(*cell, FLAG_ITEM, mode);
+                       } else {
+                               MathAtom at = MathAtom(new InsetMathMacro(buf, t.cs()));
+                               cell->push_back(at);
+                               cell->push_back(MathAtom(new InsetMathBrace(count)));
+                       }
                }
-#endif
 
                else if (t.cs() == "limits" || t.cs() == "nolimits") {
-                       CatCode const cat = nextToken().cat();
-                       if (cat == catSuper || cat == catSub)
-                               limits = t.cs() == "limits" ? 1 : -1;
+                       if (!cell->empty())
+                               cell->back()->limits(t.cs() == "limits" ? LIMITS : NO_LIMITS);
                        else {
                                MathAtom at = createInsetMath(t.cs(), buf);
                                cell->push_back(at);
@@ -1377,7 +1419,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        grid.asHullInset()->numbered(cellrow, true);
 
                else if (t.cs() == "hline") {
-                       grid.rowinfo(cellrow).lines++;
+                       grid.rowinfo(cellrow).lines++;
                }
 
                else if (t.cs() == "sqrt") {
@@ -1385,12 +1427,10 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        parse(ar, FLAG_OPTION, mode);
                        if (!ar.empty()) {
                                cell->push_back(MathAtom(new InsetMathRoot(buf)));
-                               cell->back().nucleus()->cell(0) = ar;
-                               parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
-                       } else {
+                               cell->back().nucleus()->cell(1) = ar;
+                       } else
                                cell->push_back(MathAtom(new InsetMathSqrt(buf)));
-                               parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
-                       }
+                       parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
                }
 
                else if (t.cs() == "cancelto") {
@@ -1432,7 +1472,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                else if (t.cs() == "cfrac") {
                        // allowed formats are \cfrac[pos]{num}{denom}
                        docstring const arg = getArg('[', ']');
-                       //lyxerr << "got so far: '" << arg << "'" << endl;                              
+                       //lyxerr << "got so far: '" << arg << "'" << endl;
                                if (arg == "l")
                                        cell->push_back(MathAtom(new InsetMathFrac(buf, InsetMathFrac::CFRACLEFT)));
                                else if (arg == "r")
@@ -1456,7 +1496,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                if (ar[i].size() == 1)
                                        script[i] = ar[i][0].nucleus()->asScriptInset();
                        }
-                       bool const hasscript[2] = {script[0] ? true : false, script[1] ? true : false};
+                       bool const hasscript[2] = {script[0] != nullptr, script[1] != nullptr};
                        cell->push_back(MathAtom(new InsetMathSideset(buf, hasscript[0], hasscript[1])));
                        if (hasscript[0]) {
                                if (script[0]->hasDown())
@@ -1482,8 +1522,8 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        cell->push_back(MathAtom(new InsetMathStackrel(buf, !ar.empty())));
                        if (!ar.empty())
                                cell->back().nucleus()->cell(2) = ar;
-                       parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
                        parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
+                       parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
                }
 
                else if (t.cs() == "xrightarrow" || t.cs() == "xleftarrow") {
@@ -1492,8 +1532,22 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
                }
 
+               else if (t.cs() == "xhookrightarrow" || t.cs() == "xhookleftarrow" ||
+                            t.cs() == "xRightarrow" || t.cs() == "xLeftarrow" ||
+                                t.cs() == "xleftrightarrow" || t.cs() == "xLeftrightarrow" ||
+                                t.cs() == "xrightharpoondown" || t.cs() == "xrightharpoonup" ||
+                                t.cs() == "xleftharpoondown" || t.cs() == "xleftharpoonup" ||
+                                t.cs() == "xleftrightharpoons" || t.cs() == "xrightleftharpoons" ||
+                                t.cs() == "xmapsto") {
+                       cell->push_back(createInsetMath(t.cs(), buf));
+                       parse(cell->back().nucleus()->cell(1), FLAG_OPTION, mode);
+                       parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
+               }
+
                else if (t.cs() == "ref" || t.cs() == "eqref" || t.cs() == "prettyref"
-                         || t.cs() == "pageref" || t.cs() == "vpageref" || t.cs() == "vref") {
+                         || t.cs() == "nameref" || t.cs() == "pageref"
+                         || t.cs() == "vpageref" || t.cs() == "vref" 
+                         || t.cs() == "formatted" || t.cs() == "labelonly") {
                        cell->push_back(MathAtom(new InsetMathRef(buf, t.cs())));
                        docstring const opt = parse_verbatim_option();
                        docstring const ref = parse_verbatim_item();
@@ -1532,13 +1586,13 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
 
                else if (t.cs() == "begin") {
                        docstring const name = getArg('{', '}');
-                       
+
                        if (name.empty()) {
                                success_ = false;
                                error("found invalid environment");
                                return success_;
                        }
-                       
+
                        environments_.push_back(name);
 
                        if (name == "array" || name == "subarray") {
@@ -1546,7 +1600,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                docstring const halign = parse_verbatim_item();
                                cell->push_back(MathAtom(new InsetMathArray(buf, name,
                                        InsetMathGrid::guessColumns(halign), 1, (char)valign[0], halign)));
-                               parse2(cell->back(), FLAG_END, mode, false);
+                               parse2(cell->back(), FLAG_END, InsetMath::MATH_MODE, false);
                        }
 
                        else if (name == "tabular") {
@@ -1687,7 +1741,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                                lyxerr << "found math environment `"
                                                       << to_utf8(name)
                                                       << "' in symbols file with unsupported inset `"
-                                                      << to_utf8(l->inset)
+                                                      << l->inset
                                                       << "'." << endl;
                                        }
                                        // create generic environment inset
@@ -1698,7 +1752,8 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
 
                        else {
                                success_ = false;
-                               if (!(mode_ & Parse::QUIET)) {
+                               if (!(mode_ & Parse::QUIET) &&
+                                   !(mode_ & Parse::TRACKMACRO)) {
                                        dump();
                                        lyxerr << "found unknown math environment '"
                                               << to_utf8(name) << "'" << endl;
@@ -1709,12 +1764,12 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        }
                }
 
-               else if (t.cs() == "kern") {
+               else if (t.cs() == "kern" || t.cs() == "mkern") {
                        // FIXME: A hack...
                        docstring s;
                        int num_tokens = 0;
                        while (true) {
-                               Token const & t = getToken();
+                               Token const & tt = getToken();
                                ++num_tokens;
                                if (!good()) {
                                        s.clear();
@@ -1722,12 +1777,12 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                                putback();
                                        break;
                                }
-                               s += t.character();
+                               s += tt.character();
                                if (isValidLength(to_utf8(s)))
                                        break;
                        }
                        if (s.empty())
-                               cell->push_back(MathAtom(new InsetMathKern));
+                               cell->push_back(MathAtom(new InsetMathMacro(buf, t.cs())));
                        else
                                cell->push_back(MathAtom(new InsetMathKern(s)));
                }
@@ -1739,6 +1794,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        asArray(label, ar);
                        if (grid.asHullInset()) {
                                grid.asHullInset()->label(cellrow, label);
+                               grid.asHullInset()->numbered(cellrow, true);
                        } else {
                                cell->push_back(createInsetMath(t.cs(), buf));
                                cell->push_back(MathAtom(new InsetMathBrace(ar)));
@@ -1826,7 +1882,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        bool const prot =  nextToken().character() == '*';
                        if (prot)
                                getToken();
-                       docstring const name = t.cs();
+                       docstring const name = t.cs();
                        docstring const arg = parse_verbatim_item();
                        Length length;
                        if (prot && arg == "\\fill")
@@ -1836,7 +1892,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        else {
                                // Since the Length class cannot use length variables
                                // we must not create an InsetMathSpace.
-                               cell->push_back(MathAtom(new MathMacro(buf, name)));
+                               cell->push_back(MathAtom(new InsetMathMacro(buf, name)));
                                MathData ar;
                                mathed_parse_cell(ar, '{' + arg + '}', mode_);
                                cell->append(ar);
@@ -1855,7 +1911,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                        parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
                                } else {
                                        docstring const arg = parse_verbatim_item();
-                                       cell->push_back(MathAtom(new MathMacro(buf, t.cs())));
+                                       cell->push_back(MathAtom(new InsetMathMacro(buf, t.cs())));
                                        MathData ar;
                                        mathed_parse_cell(ar, '[' + opt + ']', mode_);
                                        cell->append(ar);
@@ -1877,23 +1933,6 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                        cell->push_back(createInsetMath(t.cs(), buf));
                        parse2(cell->back(), FLAG_ITEM, mode, false);
                }
-
-               // Disabled
-               else if (1 && t.cs() == "ar") {
-                       auto_ptr<InsetMathXYArrow> p(new InsetMathXYArrow);
-                       // try to read target
-                       parse(p->cell(0), FLAG_OTPTION, mode);
-                       // try to read label
-                       if (nextToken().cat() == catSuper || nextToken().cat() == catSub) {
-                               p->up_ = nextToken().cat() == catSuper;
-                               getToken();
-                               parse(p->cell(1), FLAG_ITEM, mode);
-                               //lyxerr << "read label: " << p->cell(1) << endl;
-                       }
-
-                       cell->push_back(MathAtom(p.release()));
-                       //lyxerr << "read cell: " << cell << endl;
-               }
 #endif
 
                else if (t.cs() == "lyxmathsym") {
@@ -1923,8 +1962,8 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                cmd = Encodings::fromLaTeXCommand(cmd,
                                        Encodings::MATH_CMD | Encodings::TEXT_CMD,
                                        termination, rem);
-                               for (size_t i = 0; i < cmd.size(); ++i)
-                                       cell->push_back(MathAtom(new InsetMathChar(cmd[i])));
+                               for (char_type c : cmd)
+                                       cell->push_back(MathAtom(new InsetMathChar(c)));
                                if (!rem.empty()) {
                                        char_type c = rem[0];
                                        cell->push_back(MathAtom(new InsetMathChar(c)));
@@ -1942,7 +1981,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                                BufferParams::package_off;
 
                        bool const is_user_macro = no_mhchem ||
-                               (buf && (mode_ & Parse::TRACKMACRO
+                               (buf && ((mode_ & Parse::TRACKMACRO)
                                         ? buf->usermacros.count(t.cs()) != 0
                                         : buf->getMacro(t.cs(), false) != 0));
 
@@ -1956,7 +1995,10 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                                        new InsetMathBig(t.cs(), delim)));
                                        else {
                                                cell->push_back(createInsetMath(t.cs(), buf));
-                                               putback();
+                                               // For some reason delim.empty()
+                                               // is always false here
+                                               if (delim.at(0))
+                                                       putback();
                                        }
                                }
 
@@ -1986,9 +2028,15 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                        putback();
                                }
 
+                               else if (l->inset == "underset" || l->inset == "overset") {
+                                       cell->push_back(createInsetMath(t.cs(), buf));
+                                       parse(cell->back().nucleus()->cell(1), FLAG_ITEM, mode);
+                                       parse(cell->back().nucleus()->cell(0), FLAG_ITEM, mode);
+                               }
+
                                else {
                                        MathAtom at = createInsetMath(t.cs(), buf);
-                                       for (InsetMath::idx_type i = 0; i < at->nargs(); ++i)
+                                       for (idx_type i = 0; i < at->nargs(); ++i)
                                                parse(at.nucleus()->cell(i),
                                                        FLAG_ITEM, asMode(mode, l->extra));
                                        cell->push_back(at);
@@ -2029,19 +2077,17 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                                        Encodings::MATH_CMD | Encodings::TEXT_CMD,
                                                        is_combining, termination);
                                        }
-                                       if (c) {
+                                       if (c && buf && buf->params().encoding().encodable(c)) {
                                                if (termination) {
                                                        if (nextToken().cat() == catBegin) {
                                                                getToken();
                                                                if (nextToken().cat() == catEnd) {
                                                                        getToken();
-                                                                       num_tokens += 2;
                                                                } else
                                                                        putback();
                                                        } else {
                                                                while (nextToken().cat() == catSpace) {
                                                                        getToken();
-                                                                       ++num_tokens;
                                                                }
                                                        }
                                                }
@@ -2054,7 +2100,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                }
                                if (!is_unicode_symbol) {
                                        MathAtom at = is_user_macro ?
-                                               MathAtom(new MathMacro(buf, t.cs()))
+                                               MathAtom(new InsetMathMacro(buf, t.cs()))
                                                : createInsetMath(t.cs(), buf);
                                        InsetMath::mode_type m = mode;
                                        //if (m == InsetMath::UNDECIDED_MODE)
@@ -2062,7 +2108,7 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                        if (at->currentMode() != InsetMath::UNDECIDED_MODE)
                                                m = at->currentMode();
                                        //lyxerr << "default creation: m2: " << m << endl;
-                                       InsetMath::idx_type start = 0;
+                                       idx_type start = 0;
                                        // this fails on \bigg[...\bigg]
                                        //MathData opt;
                                        //parse(opt, FLAG_OPTION, InsetMath::VERBATIM_MODE);
@@ -2070,9 +2116,10 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
                                        //      start = 1;
                                        //      at.nucleus()->cell(0) = opt;
                                        //}
-                                       for (InsetMath::idx_type i = start; i < at->nargs(); ++i) {
+                                       for (idx_type i = start; i < at->nargs(); ++i) {
                                                parse(at.nucleus()->cell(i), FLAG_ITEM, m);
-                                               skipSpaces();
+                                               if (mode == InsetMath::MATH_MODE)
+                                                       skipSpaces();
                                        }
                                        cell->push_back(at);
                                }
@@ -2081,7 +2128,6 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
 
 
                if (flags & FLAG_LEAVE) {
-                       flags &= ~FLAG_LEAVE;
                        break;
                }
        }
@@ -2093,16 +2139,25 @@ bool Parser::parse1(InsetMathGrid & grid, unsigned flags,
 } // anonymous namespace
 
 
+// FIXME This will likely need some work.
+char const * latexkeys::MathMLtype() const
+{
+       if (extra == "mathord")
+               return "mi";
+       return "mo";
+}
+
+
 bool mathed_parse_cell(MathData & ar, docstring const & str, Parse::flags f)
 {
-       return Parser(str, f, ar.buffer()).parse(ar, 0, f & Parse::TEXTMODE ?
+       return Parser(str, f, ar.buffer()).parse(ar, 0, (f & Parse::TEXTMODE) ?
                                InsetMath::TEXT_MODE : InsetMath::MATH_MODE);
 }
 
 
 bool mathed_parse_cell(MathData & ar, istream & is, Parse::flags f)
 {
-       return Parser(is, f, ar.buffer()).parse(ar, 0, f & Parse::TEXTMODE ?
+       return Parser(is, f, ar.buffer()).parse(ar, 0, (f & Parse::TEXTMODE) ?
                                InsetMath::TEXT_MODE : InsetMath::MATH_MODE);
 }
 
@@ -2124,7 +2179,7 @@ bool mathed_parse_normal(Buffer * buf, MathAtom & t, Lexer & lex,
 bool mathed_parse_normal(InsetMathGrid & grid, docstring const & str,
                         Parse::flags f)
 {
-       return Parser(str, f, &grid.buffer()).parse1(grid, 0, f & Parse::TEXTMODE ?
+       return Parser(str, f, &grid.buffer()).parse1(grid, 0, (f & Parse::TEXTMODE) ?
                        InsetMath::TEXT_MODE : InsetMath::MATH_MODE, false);
 }