]> git.lyx.org Git - lyx.git/blobdiff - src/mathed/math_parser.C
fix typo that put too many include paths for most people
[lyx.git] / src / mathed / math_parser.C
index 587c86c978505386193ac782e8fc2d0f60a1f0c6..70e827a319868badc88672d0ae707cf61bb2e4d1 100644 (file)
@@ -1,7 +1,7 @@
 /*
  *  File:        math_parser.C
  *  Purpose:     Parser for mathed
- *  Author:      Alejandro Aguilar Sierra <asierra@servidor.unam.mx> 
+ *  Author:      Alejandro Aguilar Sierra <asierra@servidor.unam.mx>
  *  Created:     January 1996
  *  Description: Parse LaTeX2e math mode code.
  *
  *   the GNU General Public Licence version 2 or later.
  */
 
-#include <config.h>
+/*
 
-#include <cctype>
+If someone desperately needs partial "structures" (such as a few cells of
+an array inset or similar) (s)he could uses the following hack as starting
+point to write some macros:
+
+  \newif\ifcomment
+  \commentfalse
+  \ifcomment
+         \def\makeamptab{\catcode`\&=4\relax}
+         \def\makeampletter{\catcode`\&=11\relax}
+    \def\b{\makeampletter\expandafter\makeamptab\bi}
+    \long\def\bi#1\e{}
+  \else
+    \def\b{}\def\e{}
+  \fi
+
+  ...
+
+  \[\begin{array}{ccc}
+   1 & 2\b & 3^2\\
+   4 & 5\e & 6\\
+   7 & 8 & 9
+  \end{array}\]
+
+*/
+
+
+#include <config.h>
 
 #ifdef __GNUG__
 #pragma implementation
 #endif
 
 #include "math_parser.h"
-#include "array.h"
 #include "math_inset.h"
 #include "math_arrayinset.h"
-#include "math_bigopinset.h"
+#include "math_braceinset.h"
+#include "math_boxinset.h"
 #include "math_charinset.h"
-#include "math_dotsinset.h"
-#include "math_decorationinset.h"
 #include "math_deliminset.h"
-#include "math_fracinset.h"
+#include "math_factory.h"
 #include "math_funcinset.h"
-#include "math_funcliminset.h"
+#include "math_kerninset.h"
 #include "math_macro.h"
 #include "math_macrotable.h"
 #include "math_macrotemplate.h"
-#include "math_matrixinset.h"
-#include "math_noglyphinset.h"
+#include "math_hullinset.h"
 #include "math_rootinset.h"
-#include "math_scriptinset.h"
 #include "math_sizeinset.h"
-#include "math_spaceinset.h"
 #include "math_sqrtinset.h"
-#include "math_stackrelinset.h"
-#include "math_symbolinset.h"
-#include "debug.h"
-#include "mathed/support.h"
+#include "math_scriptinset.h"
+#include "math_specialcharinset.h"
+#include "math_sqrtinset.h"
+#include "math_support.h"
+#include "math_xyarrowinset.h"
+
 #include "lyxlex.h"
+#include "debug.h"
+#include "support/LAssert.h"
 #include "support/lstrings.h"
 
+#include <cctype>
+#include <stack>
+#include <algorithm>
+
 using std::istream;
+using std::ostream;
+using std::ios;
 using std::endl;
+using std::stack;
+using std::fill;
+using std::vector;
+
+//#define FILEDEBUG
 
 
 namespace {
 
-// These are lexical codes, not semantic
-enum lexcode_enum {
-       LexNone,
-       LexESC,
-       LexAlpha,
-       LexBOP,         // Binary operators or relations
-       LexOpen,
-       LexClose,
-       LexComment,
-       LexArgument,
-       LexSpace,
-       LexNewLine,
-       LexOther,
-       LexMath,
-       LexSelf
-};
+bool stared(string const & s)
+{
+       string::size_type const n = s.size();
+       return n && s[n - 1] == '*';
+}
+
 
-lexcode_enum lexcode[256];  
+void add(MathArray & ar, char c, MathTextCodes code)
+{
+       ar.push_back(MathAtom(new MathCharInset(c, code)));
+}
+
+
+// These are TeX's catcodes
+enum CatCode {
+       catEscape,     // 0    backslash
+       catBegin,      // 1    {
+       catEnd,        // 2    }
+       catMath,       // 3    $
+       catAlign,      // 4    &
+       catNewline,    // 5    ^^M
+       catParameter,  // 6    #
+       catSuper,      // 7    ^
+       catSub,        // 8    _
+       catIgnore,     // 9
+       catSpace,      // 10   space
+       catLetter,     // 11   a-zA-Z
+       catOther,      // 12   none of the above
+       catActive,     // 13   ~
+       catComment,    // 14   %
+       catInvalid     // 15   <delete>
+};
 
+CatCode theCatcode[256];
 
-void mathed_parse_into(MathArray & array, unsigned flags);
 
-unsigned char getuchar(std::istream * is)
+inline CatCode catcode(unsigned char c)
 {
-       char c = 0;
-       is->get(c);
-       if (!is->good())
-               lyxerr << "The input stream is not well..." << endl;
-       
-       return static_cast<unsigned char>(c);
+       return theCatcode[c];
 }
 
-const unsigned char LM_TK_OPEN  = '{';
-const unsigned char LM_TK_CLOSE = '}';
 
 enum {
-       FLAG_BRACE      = 1 << 0,  //  A { needed              //}
-       FLAG_BRACE_LAST = 1 << 1,  //  // { Last } ends the parsing process
-       FLAG_RIGHT      = 1 << 2,  //  Next right ends the parsing process
-       FLAG_END        = 1 << 3,  //  Next end ends the parsing process
-       FLAG_BRACE_FONT = 1 << 4,  //  // { Next } closes a font
-       FLAG_BRACK_END  = 1 << 5,  //  // [ Next ] ends the parsing process
-       FLAG_AMPERSAND  = 1 << 6,  //  Next & ends the parsing process
-       FLAG_NEWLINE    = 1 << 7,  //  Next \\ ends the parsing process
-       FLAG_ITEM       = 1 << 8,  //  read a (possibly braced token)
-       FLAG_LEAVE      = 1 << 9,  //  marker for leaving the 
-       FLAG_OPTARG     = 1 << 10  //  reads an argument in []
+       FLAG_BRACE_LAST = 1 << 1,  //  last closing brace ends the parsing process
+       FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
+       FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
+       FLAG_BRACK_END  = 1 << 4,  //  next closing bracket ends the parsing process
+       FLAG_BOX        = 1 << 5,  //  we are in a box
+       FLAG_ITEM       = 1 << 6,  //  read a (possibly braced token)
+       FLAG_BLOCK      = 1 << 7,  //  next block ends the parsing process
+       FLAG_BLOCK2     = 1 << 8,  //  next block2 ends the parsing process
+       FLAG_LEAVE      = 1 << 9   //  leave the loop at the end
 };
 
-///
-struct {
+
+void catInit()
+{
+       fill(theCatcode, theCatcode + 256, catOther);
+       fill(theCatcode + 'a', theCatcode + 'z' + 1, catLetter);
+       fill(theCatcode + 'A', theCatcode + 'Z' + 1, catLetter);
+
+       theCatcode['\\'] = catEscape;
+       theCatcode['{']  = catBegin;
+       theCatcode['}']  = catEnd;
+       theCatcode['$']  = catMath;
+       theCatcode['&']  = catAlign;
+       theCatcode['\n'] = catNewline;
+       theCatcode['#']  = catParameter;
+       theCatcode['^']  = catSuper;
+       theCatcode['_']  = catSub;
+       theCatcode['\7f'] = catIgnore;
+       theCatcode[' ']  = catSpace;
+       theCatcode['\t'] = catSpace;
+       theCatcode['\r'] = catSpace;
+       theCatcode['~']  = catActive;
+       theCatcode['%']  = catComment;
+}
+
+
+
+//
+// Helper class for parsing
+//
+
+class Token {
+public:
        ///
-       int i;
+       Token() : cs_(), char_(0), cat_(catIgnore) {}
        ///
-       latexkeys const * l;
-} yylval;
+       Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
+       ///
+       Token(string const & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
 
+       ///
+       string const & cs() const { return cs_; }
+       ///
+       CatCode cat() const { return cat_; }
+       ///
+       char character() const { return char_; }
+       ///
+       string asString() const;
+       ///
+       bool isCR() const;
 
+private:
+       ///
+       string cs_;
+       ///
+       char char_;
+       ///
+       CatCode cat_;
+};
 
-string yytext;
-int yylineno;
-istream * yyis;
+bool Token::isCR() const
+{
+       return cs_ == "\\" || cs_ == "cr" || cs_ == "crcr";
+}
 
+string Token::asString() const
+{
+       return cs_.size() ? cs_ : string(1, char_);
+}
+
+// Angus' compiler says these are not needed
+//bool operator==(Token const & s, Token const & t)
+//{
+//     return s.character() == t.character()
+//             && s.cat() == t.cat() && s.cs() == t.cs();
+//}
+//
+//bool operator!=(Token const & s, Token const & t)
+//{
+//     return !(s == t);
+//}
+
+ostream & operator<<(ostream & os, Token const & t)
+{
+       if (t.cs().size())
+               os << "\\" << t.cs();
+       else
+               os << "[" << t.character() << "," << t.cat() << "]";
+       return os;
+}
 
 
-struct latex_mathenv_type {
-       char const *      name;
-       char const *      basename;
-       MathInsetTypes    typ;
-       bool              numbered;
-       bool              ams;
-};
+class Parser {
 
-latex_mathenv_type latex_mathenv[] = { 
-       {"math",         "math",         LM_OT_SIMPLE,   0, 0},
-       {"equation*",    "equation",     LM_OT_EQUATION, 0, 0},
-       {"equation",     "equation",     LM_OT_EQUATION, 1, 0},
-       {"eqnarray*",    "eqnarray",     LM_OT_EQNARRAY, 0, 0},
-       {"eqnarray",     "eqnarray",     LM_OT_EQNARRAY, 1, 0},
-       {"align*",       "align",        LM_OT_ALIGN,    0, 1},
-       {"align",        "align",        LM_OT_ALIGN,    1, 1},
-       {"alignat*",     "alignat",      LM_OT_ALIGNAT,  0, 1},
-       {"alignat",      "alignat",      LM_OT_ALIGNAT,  1, 1},
-       {"multline*",    "multline",     LM_OT_MULTLINE, 0, 1},
-       {"multline",     "multline",     LM_OT_MULTLINE, 1, 1},
-       {"array",        "array",        LM_OT_MATRIX,   0, 1}
+public:
+       ///
+       Parser(LyXLex & lex);
+       ///
+       Parser(istream & is);
+
+       ///
+       bool parse_macro(string & name);
+       ///
+       bool parse_normal(MathAtom &);
+       ///
+       void parse_into(MathArray & array, unsigned flags, MathTextCodes = LM_TC_MIN);
+       ///
+       int lineno() const { return lineno_; }
+       ///
+       void putback();
+
+private:
+       ///
+       void parse_into1(MathArray & array, unsigned flags, MathTextCodes);
+       ///
+       string getArg(char lf, char rf);
+       ///
+       char getChar();
+       ///
+       void error(string const & msg);
+       ///
+       bool parse_lines(MathAtom & t, bool numbered, bool outmost);
+       /// parses {... & ... \\ ... & ... }
+       bool parse_lines2(MathAtom & t, bool braced);
+       /// dump contents to screen
+       void dump() const;
+
+private:
+       ///
+       void tokenize(istream & is);
+       ///
+       void tokenize(string const & s);
+       ///
+       void skipSpaceTokens(istream & is, char c);
+       ///
+       void push_back(Token const & t);
+       ///
+       void pop_back();
+       ///
+       Token const & prevToken() const;
+       ///
+       Token const & nextToken() const;
+       ///
+       Token const & getToken();
+       /// skips spaces if any
+       void skipSpaces();
+       /// skips opening brace
+       void skipBegin();
+       /// skips closing brace
+       void skipEnd();
+       /// counts a sequence of hlines
+       int readHLines();
+       ///
+       void lex(string const & s);
+       ///
+       bool good() const;
+
+       ///
+       int lineno_;
+       ///
+       vector<Token> tokens_;
+       ///
+       unsigned pos_;
+       ///
+       bool   curr_num_;
+       ///
+       string curr_label_;
+       ///
+       string curr_skip_;
 };
 
-int const latex_mathenv_num = sizeof(latex_mathenv)/sizeof(latex_mathenv[0]);
 
+Parser::Parser(LyXLex & lexer)
+       : lineno_(lexer.getLineNo()), pos_(0), curr_num_(false)
+{
+       tokenize(lexer.getStream());
+       lexer.eatLine();
+}
+
+
+Parser::Parser(istream & is)
+       : lineno_(0), pos_(0), curr_num_(false)
+{
+       tokenize(is);
+}
+
+
+void Parser::push_back(Token const & t)
+{
+       tokens_.push_back(t);
+}
 
 
-void mathPrintError(string const & msg) 
+void Parser::pop_back()
 {
-       //lyxerr[Debug::MATHED] << "Line ~" << yylineno << ": Math parse error: " << msg << endl;
-       lyxerr << "Line ~" << yylineno << ": Math parse error: " << msg << endl;
+       tokens_.pop_back();
 }
 
 
-void lexInit()
+Token const & Parser::prevToken() const
 {
-       for (int i = 0; i <= 255; ++i) {
-               if (isdigit(i))
-                       lexcode[i] = LexOther;
-               else if (isspace(i))
-                       lexcode[i] = LexSpace;
-               else
-                       lexcode[i] = LexAlpha;
+       static const Token dummy;
+       return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
+}
+
+
+Token const & Parser::nextToken() const
+{
+       static const Token dummy;
+       return good() ? tokens_[pos_] : dummy;
+}
+
+
+Token const & Parser::getToken()
+{
+       static const Token dummy;
+       //lyxerr << "looking at token " << tokens_[pos_] << " pos: " << pos_ << '\n';
+       return good() ? tokens_[pos_++] : dummy;
+}
+
+
+void Parser::skipSpaces()
+{
+       while (nextToken().cat() == catSpace)
+               getToken();
+}
+
+
+void Parser::skipBegin()
+{
+       if (nextToken().cat() == catBegin)
+               getToken();
+       else
+               lyxerr << "'{' expected\n";
+}
+
+
+void Parser::skipEnd()
+{
+       if (nextToken().cat() == catEnd)
+               getToken();
+       else
+               lyxerr << "'}' expected\n";
+}
+
+
+int Parser::readHLines()
+{
+       int num = 0;
+       skipSpaces();
+       while (nextToken().cs() == "hline") {
+               getToken();
+               ++num;
+               skipSpaces();
        }
-       
-       lexcode['\t'] = lexcode['\f'] = lexcode[' '] = LexSpace;
-       lexcode['\n'] = LexNewLine;
-       lexcode['%'] = LexComment;
-       lexcode['#'] = LexArgument;
-       lexcode['$'] = LexMath;
-       lexcode['+'] = lexcode['-'] = lexcode['*'] = lexcode['/']
-               = lexcode['<'] = lexcode['>'] = lexcode['='] = LexBOP;
-       
-       lexcode['('] = lexcode[')'] = lexcode['|'] = lexcode['.'] =
-               lexcode['?'] = LexOther; 
-       
-       lexcode['\''] = lexcode['@'] = LexAlpha;
-       
-       lexcode['['] = lexcode[']'] = lexcode['^'] = lexcode['_'] = 
-               lexcode['&'] = LexSelf;  
-       
-       lexcode['\\'] = LexESC;
-       lexcode['{'] = LexOpen;
-       lexcode['}'] = LexClose;
+       return num;
+}
+
+
+void Parser::putback()
+{
+       --pos_;
+}
+
+
+bool Parser::good() const
+{
+       return pos_ < tokens_.size();
 }
 
 
-string lexArg(unsigned char lf, bool accept_spaces = false)
+char Parser::getChar()
 {
+       if (!good())
+               lyxerr << "The input stream is not well..." << endl;
+       return tokens_[pos_++].character();
+}
+
+
+string Parser::getArg(char left, char right)
+{
+       skipSpaces();
+
        string result;
-       unsigned char c = 0;
-       while (yyis->good()) {
-               c = getuchar(yyis);
-               if (!isspace(c))
+       char c = getChar();
+
+       if (c != left)
+               putback();
+       else
+               while ((c = getChar()) != right && good())
+                       result += c;
+
+       return result;
+}
+
+
+void Parser::tokenize(istream & is)
+{
+       // eat everything up to the next \end_inset or end of stream
+       // and store it in s for further tokenization
+       string s;
+       char c;
+       while (is.get(c)) {
+               s += c;
+               if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
+                       s = s.substr(0, s.size() - 10);
                        break;
+               }
        }
 
-       if (c != lf) {
-               yyis->putback(c);
-               return result;
-       }
-               
-       unsigned char rg = 0;
-       if (lf == '{') rg = '}';
-       if (lf == '[') rg = ']';
-       if (lf == '(') rg = ')';
-       if (!rg) {
-               lyxerr[Debug::MATHED] << "Math parse error: unknown bracket '"
-                       << lf << "'" << endl;
-               return result;
-       }
+       // tokenize buffer
+       tokenize(s);
+}
 
-       int depth = 1;
-       do {
-               unsigned char c = getuchar(yyis);
-               if (c == lf)
-                       ++depth;
-               if (c == rg)
-                       --depth;
-               if ((!isspace(c) || (c == ' ' && accept_spaces)) && depth > 0)
-                       result += c;
-       } while (depth > 0 && yyis->good());
 
-       return result;
+void Parser::skipSpaceTokens(istream & is, char c)
+{
+       // skip trailing spaces
+       while (catcode(c) == catSpace || catcode(c) == catNewline)
+               if (!is.get(c))
+                       break;
+       //lyxerr << "putting back: " << c << "\n";
+       is.putback(c);
 }
 
 
-int yylex()
+void Parser::tokenize(string const & buffer)
 {
        static bool init_done = false;
-       
+
        if (!init_done) {
-               lexInit();
+               catInit();
                init_done = true;
        }
-       
-       while (yyis->good()) {
-               unsigned char c = getuchar(yyis);
-               //lyxerr << "reading byte: '" << c << "' code: " << lexcode[c] << endl;
-               
-               if (lexcode[c] == LexNewLine) {
-                       ++yylineno; 
-                       continue;
-               } else if (lexcode[c] == LexComment) {
-                       do {
-                               c = getuchar(yyis);
-                       } while (c != '\n' && yyis->good());  // eat comments
-               } else if (lexcode[c] == LexOther) {
-                       yylval.i = c;
-                       return LM_TK_STR;
-               } else if (lexcode[c] == LexAlpha || lexcode[c] == LexSpace) {
-                       yylval.i = c;
-                       return LM_TK_ALPHA;
-               } else if (lexcode[c] == LexBOP) {
-                       yylval.i = c;
-                       return LM_TK_BOP;
-               } else if (lexcode[c] == LexMath) {
-                       yylval.i = 0;
-                       return LM_TK_MATH;
-               } else if (lexcode[c] == LexSelf) {
-                       return c;
-               } else if (lexcode[c] == LexArgument) {
-                       c = getuchar(yyis);
-                       yylval.i = c - '0';
-                       return LM_TK_ARGUMENT; 
-               } else if (lexcode[c] == LexOpen) {
-                       return LM_TK_OPEN;
-               } else if (lexcode[c] == LexClose) {
-                       return LM_TK_CLOSE;
-               } else if (lexcode[c] == LexESC)   {
-                       c = getuchar(yyis);
-                       //lyxerr << "reading second byte: '" << c << "' code: " << lexcode[c] << endl;
-                       string s;
-                       s += c;
-                       latexkeys const * l = in_word_set(s);
-                       if (l) {
-                               //lyxerr << "found key: " << l << endl;
-                               //lyxerr << "found key name: " << l->name << endl;
-                               //lyxerr << "found key token: " << l->token << endl;
-                               yylval.l = l;
-                               yylval.i = l->id;
-                               return l->token;
+
+       istringstream is(buffer.c_str(), ios::in | ios::binary);
+
+       char c;
+       while (is.get(c)) {
+               //lyxerr << "reading c: " << c << "\n";
+
+               switch (catcode(c)) {
+                       case catNewline: {
+                               ++lineno_;
+                               is.get(c);
+                               if (catcode(c) == catNewline)
+                                       ; //push_back(Token("par"));
+                               else {
+                                       push_back(Token(' ', catSpace));
+                                       is.putback(c);
+                               }
+                               break;
                        }
-                       if (lexcode[c] == LexAlpha) {
-                               yytext.erase();
-                               while (lexcode[c] == LexAlpha && yyis->good()) {
-                                       yytext += c;
-                                       c = getuchar(yyis);
+
+                       case catComment: {
+                               while (is.get(c) && catcode(c) != catNewline)
+                                       ;
+                               ++lineno_;
+                               break;
+                       }
+
+                       case catEscape: {
+                               is.get(c);
+                               if (!is) {
+                                       error("unexpected end of input");
+                               } else {
+                                       string s(1, c);
+                                       if (catcode(c) == catLetter) {
+                                               // collect letters
+                                               while (is.get(c) && catcode(c) == catLetter)
+                                                       s += c;
+                                               skipSpaceTokens(is, c);
+                                       }
+                                       push_back(Token(s));
                                }
-                               while (lexcode[c] == LexSpace && yyis->good()) 
-                                       c = getuchar(yyis);
-                               if (lexcode[c] != LexSpace)
-                                       yyis->putback(c);
-                       
-                               //lyxerr[Debug::MATHED] << "reading: text '" << yytext << "'\n";
-                               //lyxerr << "reading: text '" << yytext << "'\n";
-                               latexkeys const * l = in_word_set(yytext);
-                               if (!l) 
-                                       return LM_TK_UNDEF;
-
-                               if (l->token == LM_TK_BEGIN || l->token == LM_TK_END) { 
-                                       string name = lexArg('{');
-                                       int i = 0;
-                                       while (i < latex_mathenv_num && name != latex_mathenv[i].name)
-                                                ++i;
-                                       yylval.i = i;
-                               } else if (l->token == LM_TK_SPACE) 
-                                       yylval.i = l->id;
-                               else
-                                       yylval.l = l;
-                               return l->token;
+                               break;
+                       }
+
+                       case catSuper:
+                       case catSub: {
+                               push_back(Token(c, catcode(c)));
+                               is.get(c);
+                               skipSpaceTokens(is, c);
+                               break;
+                       }
+
+                       case catIgnore: {
+                               lyxerr << "ignoring a char: " << int(c) << "\n";
+                               break;
                        }
+
+                       default:
+                               push_back(Token(c, catcode(c)));
                }
        }
-       return 0;
+
+#ifdef FILEDEBUG
+       dump();
+#endif
 }
 
 
-MathScriptInset * prevScriptInset(MathArray const & array)
+void Parser::dump() const
 {
-       MathInset * p = array.back();
-       return (p && p->isScriptInset()) ? static_cast<MathScriptInset *>(p) : 0;
+       lyxerr << "\nTokens: ";
+       for (unsigned i = 0; i < tokens_.size(); ++i) {
+               if (i == pos_)
+                       lyxerr << " <#> ";
+               lyxerr << tokens_[i];
+       }
+       lyxerr << "\n";
 }
 
 
-MathInset * lastScriptInset(MathArray & array, bool up, bool down, int limits)
+void Parser::error(string const & msg)
 {
-       MathScriptInset * p = prevScriptInset(array);
-       if (!p) {
-               MathInset * b = array.back();
-               if (b && b->isScriptable()) {
-                       p = new MathScriptInset(up, down, b->clone());
-                       array.pop_back();       
-               } else {
-                       p = new MathScriptInset(up, down);
-               }
-               array.push_back(p);
-       }
-       if (up)
-               p->up(true);
-       if (down)
-               p->down(down);
-       if (limits)
-               p->limits(limits);
-       return p;
+       lyxerr << "Line ~" << lineno_ << ": Math parse error: " << msg << endl;
+       dump();
+       //exit(1);
 }
 
 
-static bool   curr_num;
-static string curr_label;
 
-void mathed_parse_lines(MathGridInset * p, int col, bool numbered, bool outmost)
+bool Parser::parse_lines(MathAtom & t, bool numbered, bool outmost)
 {
+       MathGridInset * p = t->asGridInset();
+       if (!p) {
+               dump();
+               lyxerr << "error in Parser::parse_lines() 1\n";
+               return false;
+       }
+
        // save global variables
-       bool   const saved_num   = curr_num;
-       string const saved_label = curr_label;
+       bool   const saved_num   = curr_num_;
+       string const saved_label = curr_label_;
+
+       // read initial hlines
+       p->rowinfo(0).lines_ = readHLines();
 
        for (int row = 0; true; ++row) {
                // reset global variables
-               curr_num   = numbered;
-               curr_label.erase();
+               curr_num_   = numbered;
+               curr_label_.erase();
 
                // reading a row
-               int idx = p->nargs() - p->ncols();
-               for (int i = 0; i < col - 1; ++i, ++idx)
-                       mathed_parse_into(p->cell(idx), FLAG_AMPERSAND);
-               mathed_parse_into(p->cell(idx), FLAG_NEWLINE | FLAG_END);
+               for (MathInset::col_type col = 0; true; ++col) {
+                       //lyxerr << "reading cell " << row << " " << col << " "
+                       // << p->ncols() << "\n";
+                       //lyxerr << "ncols: " << p->ncols() << "\n";
+
+                       if (col >= p->ncols()) {
+                               //lyxerr << "adding col " << col << "\n";
+                               p->addCol(p->ncols());
+                       }
+
+                       MathArray & ar = p->cell(col + row * p->ncols());
+                       parse_into(ar, FLAG_BLOCK);
+                       // remove 'unnecessary' braces:
+                       if (ar.size() == 1 && ar.back()->asBraceInset())
+                               ar = ar.back()->asBraceInset()->cell(0);
+                       //lyxerr << "ar: " << ar << "\n";
+
+                       // break if cell is not followed by an ampersand
+                       if (nextToken().cat() != catAlign) {
+                               //lyxerr << "less cells read than normal in row/col: "
+                               //      << row << " " << col << "\n";
+                               break;
+                       }
+
+                       // skip the ampersand
+                       getToken();
+               }
 
                if (outmost) {
-                       MathMatrixInset * m = static_cast<MathMatrixInset *>(p);
-                       m->numbered(row, curr_num);
-                       m->label(row, curr_label);
+                       MathHullInset * m = t->asHullInset();
+                       if (!m) {
+                               lyxerr << "error in Parser::parse_lines() 2\n";
+                               return false;
+                       }
+                       m->numbered(row, curr_num_);
+                       m->label(row, curr_label_);
+                       if (curr_skip_.size()) {
+                               m->vcrskip(LyXLength(curr_skip_), row);
+                               curr_skip_.erase();
+                       }
                }
 
-#ifdef WITH_WARNINGS
-#warning Hack!
-#endif
-               // no newline
-               if (yylval.i != -1)
+               // is a \\ coming?
+               if (nextToken().isCR()) {
+                       // skip the cr-token
+                       getToken();
+
+                       // try to read a length
+                       //get
+
+                       // read hlines for next row
+                       p->rowinfo(row + 1).lines_ = readHLines();
+               }
+
+               // we are finished if the next token is an 'end'
+               if (nextToken().cs() == "end") {
+                       // skip the end-token
+                       getToken();
+                       getArg('{','}');
+
+                       // leave the 'read a line'-loop
                        break;
+               }
 
+               // otherwise, we have to start a new row
                p->appendRow();
        }
 
-       // restore global variables
-       curr_num   = saved_num;
-       curr_label = saved_label;
+       // restore "global" variables
+       curr_num_   = saved_num;
+       curr_label_ = saved_label;
+
+       return true;
 }
 
 
-MathMacroTemplate * mathed_parse_macro()
+bool Parser::parse_lines2(MathAtom & t, bool braced)
 {
-       if (yylex() != LM_TK_NEWCOMMAND) {
-               lyxerr << "\\newcommand expected\n";
-               return 0;
+       MathGridInset * p = t->asGridInset();
+       if (!p) {
+               lyxerr << "error in Parser::parse_lines() 1\n";
+               return false;
        }
 
-       string name = lexArg('{').substr(1);
-       string arg  = lexArg('[');
-       int    narg = arg.empty() ? 0 : atoi(arg.c_str()); 
-       MathMacroTemplate * p = new MathMacroTemplate(name, narg);
-       mathed_parse_into(p->cell(0), FLAG_BRACE | FLAG_BRACE_LAST);
-       return p;
+       for (int row = 0; true; ++row) {
+               // reading a row
+               for (MathInset::col_type col = 0; true; ++col) {
+                       //lyxerr << "reading cell " << row << " " << col << " " << p->ncols() << "\n";
+
+                       if (col >= p->ncols()) {
+                               //lyxerr << "adding col " << col << "\n";
+                               p->addCol(p->ncols());
+                       }
+
+                       parse_into(p->cell(col + row * p->ncols()), FLAG_BLOCK2);
+                       //lyxerr << "read cell: " << p->cell(col + row * p->ncols()) << "\n";
+
+                       // break if cell is not followed by an ampersand
+                       if (nextToken().cat() != catAlign) {
+                               //lyxerr << "less cells read than normal in row/col: " << row << " " << col << "\n";
+                               break;
+                       }
+
+                       // skip the ampersand
+                       getToken();
+               }
+
+               // is a \\ coming?
+               if (nextToken().isCR()) {
+                       // skip the cr-token
+                       getToken();
+               }
+
+               // we are finished if the next token is the one we expected
+               // skip the end-token
+               // leave the 'read a line'-loop
+               if (braced) {
+                       if (nextToken().cat() == catEnd) {
+                               getToken();
+                               break;
+                       }
+               } else {
+                       if (nextToken().cs() == "end") {
+                               getToken();
+                               getArg('{','}');
+                               break;
+                       }
+               }
+
+               // otherwise, we have to start a new row
+               p->appendRow();
+       }
+
+       return true;
 }
 
 
-MathMatrixInset * mathed_parse_normal()
-{
-       MathMatrixInset * p = 0;
-       int t = yylex();
 
-       switch (t) {
-               case LM_TK_MATH:
-               case LM_TK_BEGIN: {
-                       int i = yylval.i;
-                       lyxerr[Debug::MATHED]
-                               << "reading math environment " << i << " "
-                               << latex_mathenv[i].name << "\n";
 
-                       MathInsetTypes typ = latex_mathenv[i].typ;
-                       p = new MathMatrixInset(typ);
+bool Parser::parse_macro(string & name)
+{
+       int nargs = 0;
+       name = "{error}";
+       skipSpaces();
 
-                       switch (typ) {
+       if (nextToken().cs() == "def") {
 
-                               case LM_OT_SIMPLE: {
-                                       curr_num   = latex_mathenv[i].numbered;
-                                       curr_label.erase();
-                                       mathed_parse_into(p->cell(0), 0);
-                                       p->numbered(0, curr_num);
-                                       p->label(0, curr_label);
-                                       break;
-                               }
+               getToken();
+               name = getToken().cs();
 
-                               case LM_OT_EQUATION: {
-                                       curr_num   = latex_mathenv[i].numbered;
-                                       curr_label.erase();
-                                       mathed_parse_into(p->cell(0), FLAG_END);
-                                       p->numbered(0, curr_num);
-                                       p->label(0, curr_label);
-                                       break;
-                               }
+               string pars;
+               while (good() && nextToken().cat() != catBegin)
+                       pars += getToken().cs();
+       
+               if (!good()) {
+                       lyxerr << "bad stream in parse_macro\n";
+                       dump();
+                       return false;
+               }
+                       
+               //lyxerr << "read \\def parameter list '" << pars << "'\n";
+               if (!pars.empty()) {
+                       lyxerr << "can't handle non-empty parameter lists\n";
+                       dump();
+                       return false;
+               }
 
-                               case LM_OT_EQNARRAY: {
-                                       mathed_parse_lines(p, 3, latex_mathenv[i].numbered, true);
-                                       break;
-                               }
+       } else if (nextToken().cs() == "newcommand") {
 
-                               case LM_OT_ALIGN: {
-                                       p->halign(lexArg('{'));
-                                       mathed_parse_lines(p, 2, latex_mathenv[i].numbered, true);
-                                       break;
-                               }
+               getToken();
 
-                               case LM_OT_ALIGNAT: {
-                                       p->halign(lexArg('{'));
-                                       mathed_parse_lines(p, 2, latex_mathenv[i].numbered, true);
-                                       break;
-                               }
+               if (getToken().cat() != catBegin) {
+                       lyxerr << "'{' in \\newcommand expected (1) \n";
+                       dump();
+                       return false;
+               }
 
-                               default: 
-                                       lyxerr[Debug::MATHED]
-                                               << "1: unknown math environment: " << typ << "\n";
-                       }
+               name = getToken().cs();
 
-                       break;
+               if (getToken().cat() != catEnd) {
+                       lyxerr << "'}' expected\n";
+                       return false;
                }
-               
-               default:
-                       lyxerr[Debug::MATHED]
-                               << "2 unknown math environment: " << t << "\n";
+
+               string arg  = getArg('[', ']');
+               if (!arg.empty())
+                       nargs = atoi(arg.c_str());
+
+       } else { 
+               lyxerr << "\\newcommand or \\def  expected\n";
+               return false;
+       }
+
+
+       if (getToken().cat() != catBegin) {
+               lyxerr << "'{' in macro definition expected (2)\n";
+               return false;
+       }
+
+       MathArray ar1;
+       parse_into(ar1, FLAG_BRACE_LAST);
+
+       // we cannot handle recursive stuff at all
+       MathArray test;
+       test.push_back(createMathInset(name));
+       if (ar1.contains(test)) {
+               lyxerr << "we cannot handle recursive macros at all.\n";
+               return false;
        }
 
-       return p;
+       MathArray ar2;
+       parse_into(ar2, FLAG_ITEM);
+
+       MathMacroTable::create(name, nargs, ar1, ar2);
+       return true;
 }
 
 
-latexkeys const * read_delim()
+bool Parser::parse_normal(MathAtom & matrix)
 {
-       int ld = yylex();
-       //lyxerr << "found symbol: " << ld << "\n";
-       latexkeys const * l = in_word_set(".");
-       switch (ld) {
-               case LM_TK_SYM:
-               case LM_TK_NOGLYPH:
-               case LM_TK_SPECIAL:
-               case LM_TK_BEGIN:
-                       l = yylval.l;
-                       //lyxerr << "found key 1: '" << l << "'\n";
-                       //lyxerr << "found key 1: '" << l->name << "'\n";
-                       break;
-               case LM_TK_STR:
-                       string s;
-                       s += yylval.i;
-                       l = in_word_set(s);
-                       //lyxerr << "found key 2: '" << l->name << "'\n";
+       skipSpaces();
+       Token const & t = getToken();
+
+       if (t.cs() == "(") {
+               matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
+               parse_into(matrix->cell(0), 0);
+               return true;
        }
-       return l;
+
+       if (t.cat() == catMath) {
+               Token const & n = getToken();
+               if (n.cat() == catMath) {
+                       // TeX's $$...$$ syntax for displayed math
+                       matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
+                       MathHullInset * p = matrix->asHullInset();
+                       parse_into(p->cell(0), 0);
+                       p->numbered(0, curr_num_);
+                       p->label(0, curr_label_);
+               } else {
+                       // simple $...$  stuff
+                       putback();
+                       matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
+                       parse_into(matrix->cell(0), 0);
+               }
+               return true;
+       }
+
+       if (!t.cs().size()) {
+               lyxerr << "start of math expected, got '" << t << "'\n";
+               return false;
+       }
+
+       string const & cs = t.cs();
+
+       if (cs == "[") {
+               curr_num_ = 0;
+               curr_label_.erase();
+               matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
+               MathHullInset * p = matrix->asHullInset();
+               parse_into(p->cell(0), 0);
+               p->numbered(0, curr_num_);
+               p->label(0, curr_label_);
+               return true;
+       }
+
+       if (cs != "begin") {
+               lyxerr << "'begin' of un-simple math expected, got '" << cs << "'\n";
+               return false;
+       }
+
+       string const name = getArg('{', '}');
+
+       if (name == "math") {
+               matrix = MathAtom(new MathHullInset(LM_OT_SIMPLE));
+               parse_into(matrix->cell(0), 0);
+               return true;
+       }
+
+       if (name == "equation" || name == "equation*" || name == "displaymath") {
+               curr_num_ = (name == "equation");
+               curr_label_.erase();
+               matrix = MathAtom(new MathHullInset(LM_OT_EQUATION));
+               MathHullInset * p = matrix->asHullInset();
+               parse_into(p->cell(0), FLAG_END);
+               p->numbered(0, curr_num_);
+               p->label(0, curr_label_);
+               return true;
+       }
+
+       if (name == "eqnarray" || name == "eqnarray*") {
+               matrix = MathAtom(new MathHullInset(LM_OT_EQNARRAY));
+               return parse_lines(matrix, !stared(name), true);
+       }
+
+       if (name == "align" || name == "align*") {
+               matrix = MathAtom(new MathHullInset(LM_OT_ALIGN));
+               return parse_lines(matrix, !stared(name), true);
+       }
+
+       if (name == "alignat" || name == "alignat*") {
+               int nc = 2 * atoi(getArg('{', '}').c_str());
+               matrix = MathAtom(new MathHullInset(LM_OT_ALIGNAT, nc));
+               return parse_lines(matrix, !stared(name), true);
+       }
+
+       if (name == "xalignat" || name == "xalignat*") {
+               int nc = 2 * atoi(getArg('{', '}').c_str());
+               matrix = MathAtom(new MathHullInset(LM_OT_XALIGNAT, nc));
+               return parse_lines(matrix, !stared(name), true);
+       }
+
+       if (name == "xxalignat") {
+               int nc = 2 * atoi(getArg('{', '}').c_str());
+               matrix = MathAtom(new MathHullInset(LM_OT_XXALIGNAT, nc));
+               return parse_lines(matrix, !stared(name), true);
+       }
+
+       if (name == "multline" || name == "multline*") {
+               matrix = MathAtom(new MathHullInset(LM_OT_MULTLINE));
+               return parse_lines(matrix, !stared(name), true);
+       }
+
+       if (name == "gather" || name == "gather*") {
+               matrix = MathAtom(new MathHullInset(LM_OT_GATHER));
+               return parse_lines(matrix, !stared(name), true);
+       }
+
+       lyxerr[Debug::MATHED] << "1: unknown math environment: " << name << "\n";
+       lyxerr << "1: unknown math environment: " << name << "\n";
+       return false;
 }
 
 
-void mathed_parse_into(MathArray & array, unsigned flags)
+void Parser::parse_into(MathArray & array, unsigned flags, MathTextCodes code)
 {
-       static int plevel = -1;
+       parse_into1(array, flags, code);
+       // remove 'unnecessary' braces:
+       if (array.size() == 1 && array.back()->asBraceInset()) {
+               lyxerr << "extra braces removed\n";
+               array = array.back()->asBraceInset()->cell(0);
+       }
+}
 
-       ++plevel;
-       MathTextCodes yyvarcode   = LM_TC_VAR;
 
-       int  t      = yylex();
+void Parser::parse_into1(MathArray & array, unsigned flags, MathTextCodes code)
+{
        bool panic  = false;
-       int  brace  = 0;
        int  limits = 0;
 
-       while (t) {
-               //lyxerr << "t: " << t << " flags: " << flags << " i: " << yylval.i
-               //      << " '" << yytext << "'\n";
-               //array.dump(lyxerr);
-               //lyxerr << "\n";
+       while (good()) {
+               Token const & t = getToken();
+
+#ifdef FILEDEBUG
+               lyxerr << "t: " << t << " flags: " << flags << "\n";
+               //array.dump();
+               lyxerr << "\n";
+#endif
 
                if (flags & FLAG_ITEM) {
+                       if (t.cat() == catSpace)
+                               continue;
+
                        flags &= ~FLAG_ITEM;
-                       if (t == LM_TK_OPEN) { 
-                               // skip the brace and regard everything to the next matching
+                       if (t.cat() == catBegin) {
+                               // skip the brace and collect everything to the next matching
                                // closing brace
-                               t = yylex();
-                               ++brace;
                                flags |= FLAG_BRACE_LAST;
-                       } else {
-                               // regard only this single token
-                               flags |= FLAG_LEAVE;
+                               continue;
                        }
+
+                       // handle only this single token, leave the loop if done
+                       flags |= FLAG_LEAVE;
                }
 
-               if ((flags & FLAG_BRACE) && t != LM_TK_OPEN) {
-                       mathPrintError(
-                               "Expected {. Maybe you forgot to enclose an argument in {}");
-                       panic = true;
-                       break;
+               if (flags & FLAG_BLOCK) {
+                       if (t.cat() == catAlign || t.isCR() || t.cs() == "end") {
+                               putback();
+                               return;
+                       }
                }
 
-               switch (t) {
-                       
-               case LM_TK_ALPHA:
-                       if (!isspace(yylval.i) || yyvarcode == LM_TC_TEXTRM)
-                               array.push_back(new MathCharInset(yylval.i, yyvarcode));
-                       break;
+               if (flags & FLAG_BLOCK2) {
+                       if (t.cat() == catAlign || t.isCR() || t.cs() == "end"
+                                       || t.cat() == catEnd) {
+                               putback();
+                               return;
+                       }
+               }
 
-               case LM_TK_ARGUMENT: {
-                       MathMacroArgument * p = new MathMacroArgument(yylval.i);
-                       p->code(yyvarcode);
-                       array.push_back(p);
-                       break;
+               //
+               // cat codes
+               //
+               if (t.cat() == catMath) {
+                       if (flags & FLAG_BOX) {
+                               // we are inside an mbox, so opening new math is allowed
+                               array.push_back(MathAtom(new MathHullInset(LM_OT_SIMPLE)));
+                               parse_into(array.back()->cell(0), 0);
+                       } else {
+                               // otherwise this is the end of the formula
+                               break;
+                       }
                }
 
-               case LM_TK_SPECIAL:
-                       array.push_back(new MathCharInset(yylval.i, LM_TC_SPECIAL));
-                       break;
+               else if (t.cat() == catLetter)
+                       add(array, t.character(), code);
 
-               case LM_TK_STR:
-                       array.push_back(new MathCharInset(yylval.i, LM_TC_CONST));
-                       break;
+               else if (t.cat() == catSpace && code == LM_TC_TEXTRM)
+                       add(array, t.character(), code);
 
-               case LM_TK_OPEN:
-                       ++brace;
-                       if (flags & FLAG_BRACE)
-                               flags &= ~FLAG_BRACE;
-                       else 
-                               array.push_back(new MathCharInset('{', LM_TC_TEX));
-                       break;
+               else if (t.cat() == catParameter) {
+                       Token const & n = getToken();
+                       array.push_back(MathAtom(new MathMacroArgument(n.character()-'0', code)));
+               }
 
-               case LM_TK_CLOSE:
-                       --brace;
-                       if (brace < 0) {
-                               mathPrintError("Unmatching braces");
-                               panic = true;
-                               break;
-                       }
-                       if (flags & FLAG_BRACE_FONT) {
-                               yyvarcode = LM_TC_VAR;
-                               flags &= ~FLAG_BRACE_FONT;
-                               break;
+               else if (t.cat() == catBegin) {
+                       MathArray ar;
+                       parse_into(ar, FLAG_BRACE_LAST);
+#ifndef WITH_WARNINGS
+#warning this might be wrong in general!
+#endif
+                       // ignore braces around simple items
+                       if ((ar.size() == 1 && !ar.front()->needsBraces()
+       || (ar.size() == 2 && !ar.front()->needsBraces()
+                                           && ar.back()->asScriptInset()))
+       || (ar.size() == 0 && array.size() == 0))
+                       {
+                               array.push_back(ar);
+                       } else {
+                               array.push_back(MathAtom(new MathBraceInset));
+                               array.back()->cell(0).swap(ar);
                        }
-                       if (brace == 0 && (flags & FLAG_BRACE_LAST))
-                               flags |= FLAG_LEAVE;
-                       else
-                               array.push_back(new MathCharInset('}', LM_TC_TEX));
-                       break;
-               
-               case '[':
-                       array.push_back(new MathCharInset('[', LM_TC_CONST));
-                       break;
+               }
 
-               case ']':
-                       if (flags & FLAG_BRACK_END)
-                               flags |= FLAG_LEAVE;
-                       else 
-                               array.push_back(new MathCharInset(']', LM_TC_CONST));
-                       break;
-               
-               case '^':
-                       mathed_parse_into(
-                               lastScriptInset(array, true, false, limits)->cell(0), FLAG_ITEM);
-                       break;
-               
-               case '_':
-                       mathed_parse_into(
-                               lastScriptInset(array, false, true, limits)->cell(1), FLAG_ITEM);
-                       break;
-               
-               case LM_TK_LIMIT:
-                       limits = yylval.l->id;
-                       //lyxerr << "setting limit to " << limits << "\n";
-                       break;
-               
-               case '&':
-                       if (flags & FLAG_AMPERSAND) {
-                               flags &= ~FLAG_AMPERSAND;
-                               --plevel;
+               else if (t.cat() == catEnd) {
+                       if (flags & FLAG_BRACE_LAST)
                                return;
-                       }
-                       lyxerr[Debug::MATHED]
-                               << "found tab unexpectedly, array: '" << array << "'\n";
-                       break;
-               
-               case LM_TK_NEWLINE:
-                       if (flags & FLAG_NEWLINE) {
-                               flags &= ~FLAG_NEWLINE;
-                               --plevel;
-                               return;
-                       }
-                       lyxerr[Debug::MATHED]
-                               << "found newline unexpectedly, array: '" << array << "'\n";
-                       break;
-               
-               case LM_TK_PROTECT: 
-                       break;
+                       dump();
+                       lyxerr << "found '}' unexpectedly, array: '" << array << "'\n";
+                       //lyxerr << "found '}' unexpectedly\n";
+                       lyx::Assert(0);
+                       add(array, '}', LM_TC_TEX);
+               }
 
-               case LM_TK_NOGLYPH: 
-               case LM_TK_NOGLYPHB: 
-                       limits = 0;
-                       array.push_back(new MathNoglyphInset(yylval.l));
-                       break;
+               else if (t.cat() == catAlign) {
+                       lyxerr << "found tab unexpectedly, array: '" << array << "'\n";
+                       //lyxerr << "found tab unexpectedly\n";
+                       add(array, '&', LM_TC_TEX);
+               }
 
-               case LM_TK_BIGSYM:  
+               else if (t.cat() == catSuper || t.cat() == catSub) {
+                       bool up = (t.cat() == catSuper);
+                       MathScriptInset * p = 0;
+                       if (array.size())
+                               p = array.back()->asScriptInset();
+                       if (!p || p->has(up)) {
+                               array.push_back(MathAtom(new MathScriptInset(up)));
+                               p = array.back()->asScriptInset();
+                       }
+                       p->ensure(up);
+                       parse_into(p->cell(up), FLAG_ITEM);
+                       p->limits(limits);
                        limits = 0;
-                       array.push_back(new MathBigopInset(yylval.l));
-                       break;
+               }
 
-               case LM_TK_FUNCLIM:
-                       limits = 0;
-                       array.push_back(new MathFuncLimInset(yylval.l));
-                       break;
+               else if (t.character() == ']' && (flags & FLAG_BRACK_END))
+                       return;
 
-               case LM_TK_SYM:
-                       limits = 0;
-                       array.push_back(new MathSymbolInset(yylval.l));
-                       break;
+               else if (t.cat() == catOther)
+                       add(array, t.character(), code);
 
-               case LM_TK_BOP:
-                       array.push_back(new MathCharInset(yylval.i, LM_TC_BOP));
-                       break;
+               //
+               // control sequences
+               //
+               else if (t.cs() == "protect")
+                       // ignore \\protect, will be re-added during output
+                       ;
 
-               case LM_TK_SPACE:
-                       if (yylval.i >= 0) 
-                               array.push_back(new MathSpaceInset(yylval.i));
+               else if (t.cs() == "end")
                        break;
 
-               case LM_TK_DOTS:
-                       array.push_back(new MathDotsInset(yylval.l));
-                       break;
-               
-               case LM_TK_STACK:
-               {
-                       MathStackrelInset * p = new MathStackrelInset;
-                       mathed_parse_into(p->cell(0), FLAG_ITEM);
-                       mathed_parse_into(p->cell(1), FLAG_ITEM);
-                       array.push_back(p);
+               else if (t.cs() == ")")
                        break;
-               }
 
-               case LM_TK_FRAC:
-               {
-                       MathFracInset * p = new MathFracInset;
-                       mathed_parse_into(p->cell(0), FLAG_ITEM);
-                       mathed_parse_into(p->cell(1), FLAG_ITEM);
-                       array.push_back(p);
+               else if (t.cs() == "]")
                        break;
+
+               else if (t.cs() == "\\") {
+                       curr_skip_ = getArg('[', ']');
+                       //lyxerr << "found newline unexpectedly, array: '" << array << "'\n";
+                       lyxerr << "found newline unexpectedly\n";
+                       array.push_back(createMathInset("\\"));
                }
 
-               case LM_TK_SQRT:
-               {
-                       unsigned char c = getuchar(yyis);
+               else if (t.cs() == "limits")
+                       limits = 1;
+
+               else if (t.cs() == "nolimits")
+                       limits = -1;
+
+               else if (t.cs() == "nonumber")
+                       curr_num_ = false;
+
+               else if (t.cs() == "number")
+                       curr_num_ = true;
+
+               else if (t.cs() == "sqrt") {
+                       char c = getChar();
                        if (c == '[') {
-                               array.push_back(new MathRootInset);
-                               mathed_parse_into(array.back()->cell(0), FLAG_BRACK_END);
-                               mathed_parse_into(array.back()->cell(1), FLAG_ITEM);
+                               array.push_back(MathAtom(new MathRootInset));
+                               parse_into(array.back()->cell(0), FLAG_BRACK_END);
+                               parse_into(array.back()->cell(1), FLAG_ITEM);
                        } else {
-                               yyis->putback(c);
-                               array.push_back(new MathSqrtInset);
-                               mathed_parse_into(array.back()->cell(0), FLAG_ITEM);
+                               putback();
+                               array.push_back(MathAtom(new MathSqrtInset));
+                               parse_into(array.back()->cell(0), FLAG_ITEM);
                        }
-                       break;
                }
-               
-               case LM_TK_LEFT:
-               {
-                       latexkeys const * l = read_delim();
+
+               else if (t.cs() == "left") {
+                       string l = getToken().asString();
                        MathArray ar;
-                       mathed_parse_into(ar, FLAG_RIGHT);
-                       latexkeys const * r = read_delim();
-                       MathDelimInset * dl = new MathDelimInset(l, r);
+                       parse_into(ar, FLAG_RIGHT);
+                       string r = getToken().asString();
+                       MathAtom dl(new MathDelimInset(l, r));
                        dl->cell(0) = ar;
                        array.push_back(dl);
-                       break;
                }
-               
-               case LM_TK_RIGHT:
-                       if (flags & FLAG_RIGHT) { 
-                               --plevel;
-                               return;
+
+               else if (t.cs() == "right") {
+                       if (!(flags & FLAG_RIGHT)) {
+                               //lyxerr << "got so far: '" << array << "'\n";
+                               error("Unmatched right delimiter");
                        }
-                       mathPrintError("Unmatched right delimiter");
-//       panic = true;
-                       break;
-               
-               case LM_TK_FONT:
-                       yyvarcode = static_cast<MathTextCodes>(yylval.l->id);
-                       flags |= (FLAG_BRACE | FLAG_BRACE_FONT);
-                       break;
+                       return;
+               }
 
-               case LM_TK_STY:
-               {
-                       lyxerr[Debug::MATHED] << "LM_TK_STY not implemented\n";
-                       //MathArray tmp = array;
-                       //MathSizeInset * p = new MathSizeInset(MathStyles(yylval.l->id));
-                       //array.push_back(p);
-                       //mathed_parse_into(p->cell(0), FLAG_BRACE_FONT);
-                       break; 
+               else if (t.cs() == "begin") {
+                       string const name = getArg('{', '}');
+                       if (name == "array" || name == "subarray") {
+                               string const valign = getArg('[', ']') + 'c';
+                               string const halign = getArg('{', '}');
+                               array.push_back(MathAtom(new MathArrayInset(name, valign[0], halign)));
+                               parse_lines(array.back(), false, false);
+                       } else if (name == "split" || name == "cases" ||
+                                        name == "gathered" || name == "aligned") {
+                               array.push_back(createMathInset(name));
+                               parse_lines(array.back(), false, false);
+                       } else if (name == "matrix"  || name == "pmatrix" || name == "bmatrix" ||
+                                        name == "vmatrix" || name == "Vmatrix") {
+                               array.push_back(createMathInset(name));
+                               parse_lines2(array.back(), false);
+                       } else
+                               lyxerr << "unknow math inset begin '" << name << "'\n";
                }
 
+               else if (t.cs() == "kern") {
+#ifdef WITH_WARNINGS
+#warning A hack...
+#endif
+                       string s;
+                       while (1) {
+                               Token const & t = getToken();
+                               if (!good()) {
+                                       putback();
+                                       break;
+                               }
+                               s += t.character();
+                               if (isValidLength(s))
+                                       break;
+                       }
+                       array.push_back(MathAtom(new MathKernInset(s)));
+               }
 
-               case LM_TK_DECORATION:
-               {  
-                       MathDecorationInset * p = new MathDecorationInset(yylval.l);
-                       mathed_parse_into(p->cell(0), FLAG_ITEM);
+/*
+               else if (t.cs() == "lyxkern") {
+                       MathAtom p = createMathInset(t.cs());
+                       parse_into(p->cell(0), flags, code);
                        array.push_back(p);
-                       break;
                }
-                       
-               case LM_TK_NONUM:
-                       curr_num = false;
-                       break;
-               
-               case LM_TK_FUNC:
-                       array.push_back(new MathSymbolInset(yylval.l));
-                       break;
-               
-               case LM_TK_UNDEF: 
-                       if (MathMacroTable::hasTemplate(yytext)) {
-                               MathMacro * m = MathMacroTable::cloneTemplate(yytext);
-                               for (int i = 0; i < m->nargs(); ++i) 
-                                       mathed_parse_into(m->cell(i), FLAG_ITEM);
-                               array.push_back(m);
-                               m->metrics(LM_ST_TEXT);
-                       } else
-                               array.push_back(new MathFuncInset(yytext));
-                       break;
-               
-               case LM_TK_MATH:
-               case LM_TK_END:
-                       --plevel;
+*/
+
+               else if (t.cs() == "label") {
+                       curr_label_ = getArg('{', '}');
+               }
+
+               else if (t.cs() == "choose" || t.cs() == "over" || t.cs() == "atop") {
+                       MathAtom p = createMathInset(t.cs());
+                       array.swap(p->cell(0));
+                       parse_into(p->cell(1), flags, code);
+                       array.push_back(p);
                        return;
+               }
 
-               case LM_TK_BEGIN:
-               {
-                       int i = yylval.i;
-                       MathInsetTypes typ = latex_mathenv[i].typ;
-
-                       if (typ == LM_OT_MATRIX) {
-                               string const valign = lexArg('[') + 'c';
-                               string const halign = lexArg('{');
-                               //lyxerr << "valign: '" << valign << "'\n";
-                               //lyxerr << "halign: '" << halign << "'\n";
-                               MathArrayInset * m = new MathArrayInset(halign.size(), 1);
-                               m->valign(valign[0]);
-                               m->halign(halign);
-
-                               mathed_parse_lines(m, halign.size(), latex_mathenv[i].numbered, false);
-                               array.push_back(m);
-                               //lyxerr << "read matrix " << *m << "\n";       
-                               break;
-                       } else 
-                               lyxerr[Debug::MATHED] << "unknow math inset " << typ << "\n";   
-                       break;
+               else if (t.cs() == "substack") {
+                       array.push_back(createMathInset(t.cs()));
+                       skipBegin();
+                       parse_lines2(array.back(), true);
+               }
+
+               else if (t.cs() == "xymatrix") {
+                       array.push_back(createMathInset(t.cs()));
+                       skipBegin();
+                       parse_lines2(array.back(), true);
+               }
+
+#if 0
+               // Disabled
+               else if (1 && t.cs() == "ar") {
+                       MathXYArrowInset * p = new MathXYArrowInset;
+
+                       // try to read target
+                       char c = getChar();
+                       if (c == '[') {
+                               parse_into(p->cell(0), FLAG_BRACK_END);
+                               //lyxerr << "read target: " << p->cell(0) << "\n";
+                       } else {
+                               putback();
+                       }
+
+                       // try to read label
+                       if (nextToken().cat() == catSuper || nextToken().cat() == catSub) {
+                               p->up_ = nextToken().cat() == catSuper;
+                               getToken();
+                               parse_into(p->cell(1), FLAG_ITEM);
+                               //lyxerr << "read label: " << p->cell(1) << "\n";
+                       }
+
+                       array.push_back(MathAtom(p));
+                       //lyxerr << "read array: " << array << "\n";
+               }
+#endif
+
+#if 0
+               else if (t.cs() == "mbox" || t.cs() == "text") {
+                       //array.push_back(createMathInset(t.cs()));
+                       array.push_back(MathAtom(new MathBoxInset(t.cs())));
+                       // slurp in the argument of mbox
+
+                       MathBoxInset * p = array.back()->asBoxInset();
+                       //lyx::assert(p);
+               }
+#endif
+
+
+               else if (t.cs().size()) {
+                       latexkeys const * l = in_word_set(t.cs());
+                       if (l) {
+                               if (l->token == LM_TK_FONT) {
+                                       //lyxerr << "starting font\n";
+                                       //CatCode catSpaceSave = theCatcode[' '];
+                                       //if (l->id == LM_TC_TEXTRM) {
+                                       //      // temporarily change catcode
+                                       //      theCatcode[' '] = catLetter;
+                                       //}
+
+                                       MathArray ar;
+                                       parse_into(ar, FLAG_ITEM, static_cast<MathTextCodes>(l->id));
+                                       array.push_back(ar);
+
+                                       // undo catcode changes
+                                       ////theCatcode[' '] = catSpaceSave;
+                                       //lyxerr << "ending font\n";
+                               }
+
+                               else if (l->token == LM_TK_OLDFONT) {
+                                       code = static_cast<MathTextCodes>(l->id);
+                               }
+
+                               else if (l->token == LM_TK_BOX) {
+                                       MathAtom p = createMathInset(t.cs());
+                                       parse_into(p->cell(0), FLAG_ITEM | FLAG_BOX, LM_TC_BOX);
+                                       array.push_back(p);
+                               }
+
+                               else if (l->token == LM_TK_STY) {
+                                       MathAtom p = createMathInset(t.cs());
+                                       parse_into(p->cell(0), flags, code);
+                                       array.push_back(p);
+                                       return;
+                               }
+
+                               else {
+                                       MathAtom p = createMathInset(t.cs());
+                                       for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
+                                               parse_into(p->cell(i), FLAG_ITEM);
+                                       array.push_back(p);
+                               }
+                       }
+
+                       else {
+                               MathAtom p = createMathInset(t.cs());
+                               for (MathInset::idx_type i = 0; i < p->nargs(); ++i)
+                                       parse_into(p->cell(i), FLAG_ITEM);
+                               array.push_back(p);
+                       }
                }
-       
-               case LM_TK_MACRO:
-                       array.push_back(MathMacroTable::cloneTemplate(yylval.l->name));
-                       break;
-               
-               case LM_TK_LABEL:
-                       curr_label = lexArg('{', true);
-                       break;
-               
-               default:
-                       mathPrintError("Unrecognized token");
-                       lyxerr[Debug::MATHED] << "[" << t << " " << yytext << "]" << endl;
-                       break;
 
-               } // end of big switch
 
                if (flags & FLAG_LEAVE) {
                        flags &= ~FLAG_LEAVE;
                        break;
                }
+       }
 
-               if (panic) {
-                       lyxerr << " Math Panic, expect problems!" << endl;
-                       //   Search for the end command. 
-                       do {
-                               t = yylex();
-                       } while (yyis->good() && t != LM_TK_END && t);
-               } else {
-                       t = yylex();
-               }
+       if (panic) {
+               lyxerr << " Math Panic, expect problems!\n";
+               //   Search for the end command.
+               Token t;
+               do {
+                       t = getToken();
+               } while (good() && t.cs() != "end");
        }
-       --plevel;
 }
 
-void mathed_parse_end(LyXLex & lex)
-{
-       // Update line number
-       lex.setLineNo(yylineno);
 
-       // reading of end_inset
-       while (lex.isOK()) {
-               lex.nextToken();
-               if (lex.getString() == "\\end_inset")
-                       break;
-               lyxerr[Debug::MATHED] << "InsetFormula::Read: Garbage before \\end_inset,"
-                       " or missing \\end_inset!" << endl;
-       }
-}
 
-}
+} // anonymous namespace
 
 
-MathArray mathed_parse_cell(string const & str)
+void mathed_parse_cell(MathArray & ar, string const & str)
 {
        istringstream is(str.c_str());
-       yyis     = &is;
-       yylineno = 0;
-       MathArray ar;
-       mathed_parse_into(ar, 0);
-       return ar;
+       mathed_parse_cell(ar, is);
+}
+
+
+void mathed_parse_cell(MathArray & ar, istream & is)
+{
+       Parser(is).parse_into(ar, 0);
 }
 
 
 
-MathMacroTemplate * mathed_parse_macro(string const & str)
+bool mathed_parse_macro(string & name, string const & str)
 {
        istringstream is(str.c_str());
-       return mathed_parse_macro(is);
+       Parser parser(is);
+       return parser.parse_macro(name);
 }
 
-MathMacroTemplate * mathed_parse_macro(istream & is)
+bool mathed_parse_macro(string & name, istream & is)
 {
-       yyis     = &is;
-       yylineno = 0;
-       return mathed_parse_macro();
+       Parser parser(is);
+       return parser.parse_macro(name);
 }
 
-MathMacroTemplate * mathed_parse_macro(LyXLex & lex)
+bool mathed_parse_macro(string & name, LyXLex & lex)
 {
-       yyis     = &lex.getStream();
-       yylineno = lex.getLineNo();
-       MathMacroTemplate * p = mathed_parse_macro();
-       mathed_parse_end(lex);
-       return p;
+       Parser parser(lex);
+       return parser.parse_macro(name);
 }
 
 
 
-MathMatrixInset * mathed_parse_normal(string const & str)
+bool mathed_parse_normal(MathAtom & t, string const & str)
 {
        istringstream is(str.c_str());
-       return mathed_parse_normal(is);
+       Parser parser(is);
+       return parser.parse_normal(t);
 }
 
-MathMatrixInset * mathed_parse_normal(istream & is)
+bool mathed_parse_normal(MathAtom & t, istream & is)
 {
-       yyis     = &is;
-       yylineno = 0;
-       return mathed_parse_normal();
+       Parser parser(is);
+       return parser.parse_normal(t);
 }
 
-MathMatrixInset * mathed_parse_normal(LyXLex & lex)
+bool mathed_parse_normal(MathAtom & t, LyXLex & lex)
 {
-       yyis     = &lex.getStream();
-       yylineno = lex.getLineNo();
-       MathMatrixInset * p = mathed_parse_normal();
-       mathed_parse_end(lex);
-       return p;
+       Parser parser(lex);
+       return parser.parse_normal(t);
 }
-