]> git.lyx.org Git - lyx.git/blobdiff - src/mathed/math_parser.C
further code uglification to make Jean-Marc's compiler happy
[lyx.git] / src / mathed / math_parser.C
index 32c49d0bc89360bacd6dba01db9a2953d9a02ca6..18766700e37c3bd16dd972b404597f445f137837 100644 (file)
 
 #include "math_parser.h"
 #include "array.h"
-#include "math_rowst.h"
-#include "math_iter.h"
 #include "math_inset.h"
+#include "math_arrayinset.h"
+#include "math_charinset.h"
+#include "math_deliminset.h"
+#include "math_factory.h"
+#include "math_funcinset.h"
+#include "math_kerninset.h"
 #include "math_macro.h"
 #include "math_macrotable.h"
 #include "math_macrotemplate.h"
-#include "math_root.h"
 #include "math_matrixinset.h"
-#include "math_accentinset.h"
-#include "math_bigopinset.h"
-#include "math_funcinset.h"
-#include "math_spaceinset.h"
-#include "math_dotsinset.h"
-#include "math_fracinset.h"
-#include "math_deliminset.h"
-#include "math_decorationinset.h"
+#include "math_rootinset.h"
+#include "math_sqrtinset.h"
+#include "math_scriptinset.h"
+#include "math_specialcharinset.h"
+#include "math_splitinset.h"
+#include "math_sqrtinset.h"
 #include "debug.h"
-#include "support/lyxlib.h"
-#include "mathed/support.h"
-#include "boost/array.hpp"
+#include "support.h"
+#include "lyxlex.h"
+#include "support/lstrings.h"
 
 using std::istream;
+using std::ostream;
+using std::ios;
 using std::endl;
 
 
-extern MathMatrixInset * create_multiline(short int type, int cols);
+namespace {
+
+bool stared(string const & s)
+{
+       unsigned n = s.size();
+       return n && s[n - 1] == '*';
+}
+
+
+
+// These are TeX's catcodes
+enum CatCode {
+       catEscape,     // 0    backslash 
+       catBegin,      // 1    {
+       catEnd,        // 2    }
+       catMath,       // 3    $
+       catAlign,      // 4    &
+       catNewline,    // 5    ^^M
+       catParameter,  // 6    #
+       catSuper,      // 7    ^
+       catSub,        // 8    _
+       catIgnore,     // 9       
+       catSpace,      // 10   space
+       catLetter,     // 11   a-zA-Z
+       catOther,      // 12   none of the above
+       catActive,     // 13   ~
+       catComment,    // 14   %
+       catInvalid     // 15   <delete>
+};
+
+CatCode theCatcode[256];  
+
+
+inline CatCode catcode(unsigned char c)
+{
+       return theCatcode[c];
+}
 
 
 enum {
-       FLAG_BRACE      = 1,    //  A { needed
-       FLAG_BRACE_ARG  = 2,    //  Next { is argument
-       FLAG_BRACE_OPT  = 4,    //  Optional {
-       FLAG_BRACE_LAST = 8,    //  Last } ends the parsing process
-       FLAG_BRACK_ARG  = 16,   //  Optional [
-       FLAG_RIGHT      = 32,      //  Next right ends the parsing process
-       FLAG_END        = 64,      //  Next end ends the parsing process
-       FLAG_BRACE_FONT = 128,  //  Next } closes a font
-       FLAG_BRACK_END  = 256   // Next ] ends the parsing process
+       FLAG_BRACE      = 1 << 0,  //  an opening brace needed
+       FLAG_BRACE_LAST = 1 << 1,  //  last closing brace ends the parsing process
+       FLAG_RIGHT      = 1 << 2,  //  next \\right ends the parsing process
+       FLAG_END        = 1 << 3,  //  next \\end ends the parsing process
+       FLAG_BRACK_END  = 1 << 4,  //  next closing bracket ends the parsing process
+       FLAG_NEWLINE    = 1 << 6,  //  next \\\\ ends the parsing process
+       FLAG_ITEM       = 1 << 7,  //  read a (possibly braced token)
+       FLAG_BLOCK      = 1 << 8,  //  next block ends the parsing process
+       FLAG_LEAVE      = 1 << 9   //  leave the loop at the end
 };
 
 
-///
-union YYSTYPE {
+void catInit()
+{
+       for (int i = 0; i <= 255; ++i) 
+               theCatcode[i] = catOther;
+       for (int i = 'a'; i <= 'z'; ++i) 
+               theCatcode[i] = catLetter;
+       for (int i = 'A'; i <= 'Z'; ++i) 
+               theCatcode[i] = catLetter;
+
+       theCatcode['\\'] = catEscape;   
+       theCatcode['{']  = catBegin;    
+       theCatcode['}']  = catEnd;      
+       theCatcode['$']  = catMath;     
+       theCatcode['&']  = catAlign;    
+       theCatcode['\n'] = catNewline;  
+       theCatcode['#']  = catParameter;        
+       theCatcode['^']  = catSuper;    
+       theCatcode['_']  = catSub;      
+       theCatcode['\7f'] = catIgnore;    
+       theCatcode[' ']  = catSpace;    
+       theCatcode['\t'] = catSpace;    
+       theCatcode['\r'] = catSpace;    
+       theCatcode['~']  = catActive;   
+       theCatcode['%']  = catComment;  
+}
+
+
+
+//
+// Helper class for parsing
+//
+
+class Token {
+public:
+       ///
+       Token() : cs_(), char_(0), cat_(catIgnore) {}
+       ///
+       Token(char c, CatCode cat) : cs_(), char_(c), cat_(cat) {}
+       ///
+       Token(const string & cs) : cs_(cs), char_(0), cat_(catIgnore) {}
+
+       ///
+       string const & cs() const { return cs_; }
+       ///
+       CatCode cat() const { return cat_; }
+       ///
+       char character() const { return char_; }
        ///
-       unsigned char c;
+       string asString() const;
+
+private:       
        ///
-       char const * s;
+       string cs_;
        ///
-       int i;
+       char char_;
        ///
-       latexkeys const * l;
+       CatCode cat_;
 };
 
+string Token::asString() const
+{
+       return cs_.size() ? cs_ : string(1, char_);
+}
 
-static
-YYSTYPE yylval;
+bool operator==(Token const & s, Token const & t)
+{
+       return s.character() == t.character()
+               && s.cat() == t.cat() && s.cs() == t.cs(); 
+}
 
+bool operator!=(Token const & s, Token const & t)
+{
+       return !(s == t);
+}
 
-static
-MathedInsetTypes mathed_env = LM_OT_MIN;
+ostream & operator<<(ostream & os, Token const & t)
+{
+       if (t.cs().size())
+               os << "\\" << t.cs();
+       else
+               os << "[" << t.character() << "," << t.cat() << "]";
+       return os;
+}
 
 
-string mathed_label;
+class Parser {
 
+public:
+       ///
+       Parser(LyXLex & lex);
+       ///
+       Parser(istream & is);
 
-int const latex_mathenv_num = 12;
-char const * latex_mathenv[latex_mathenv_num] = { 
-       "math", 
-       "displaymath", 
-       "equation", 
-       "eqnarray*",
-       "eqnarray",
-       "align*",
-       "align",
-       "alignat*",
-       "alignat",
-       "multline*",
-       "multline",
-       "array"
-};
+       ///
+       MathMacroTemplate * parse_macro();
+       ///
+       MathMatrixInset * parse_normal();
+       ///
+       void parse_into(MathArray & array, unsigned flags, MathTextCodes = LM_TC_MIN);
+       ///
+       int lineno() const { return lineno_; }
+       ///
+       void putback();
 
+private:
+       ///
+       string getArg(char lf, char rf);
+       ///
+       char getChar();
+       ///
+       void error(string const & msg);
+       ///
+       void parse_lines(MathGridInset * p, bool numbered, bool outmost);
 
-char const * latex_special_chars = "#$%&_{}";
-
-
-// These are lexical codes, not semantic
-enum lexcode_enum {
-       LexNone,
-       LexESC,
-       LexAlpha,
-       LexDigit,
-       LexBOP,         // Binary operators or relations
-       LexMathSpace,
-       LexOpen,
-       LexClose,
-       LexComment,
-       LexArgument,
-       LexSpace,
-       LexNewLine,
-       LexOther,
-       LexSelf
+private:
+       ///
+       void tokenize(istream & is);
+       ///
+       void tokenize(string const & s);
+       ///
+       void push_back(Token const & t);
+       ///
+       void pop_back();
+       ///
+       Token const & prevToken() const;
+       ///
+       Token const & nextToken() const;
+       ///
+       Token const & getToken();
+       ///
+       void lex(string const & s);
+       ///
+       bool good() const;
+
+       ///
+       int lineno_;
+       ///
+       std::vector<Token> tokens_;
+       ///
+       unsigned pos_;
+       ///
+       bool   curr_num_;
+       ///
+       string curr_label_;
+       ///
+       string curr_skip_;
 };
 
 
-static lexcode_enum lexcode[256];  
-#warning Replace with string
-//static char yytext[256];
-static array<char, 256> yytext;
-static int yylineno;
-static istream * yyis;
-static bool yy_mtextmode= false;
+Parser::Parser(LyXLex & lexer)
+       : lineno_(lexer.getLineNo()), pos_(0), curr_num_(false)
+{
+       tokenize(lexer.getStream());
+       lexer.eatLine();
+}
 
 
-static inline
-void mathPrintError(string const & msg) 
+Parser::Parser(istream & is)
+       : lineno_(0), pos_(0), curr_num_(false)
 {
-       lyxerr << "Line ~" << yylineno << ": Math parse error: "
-              << msg << endl;
+       tokenize(is);
 }
 
 
-static
-void LexInitCodes()
+void Parser::push_back(Token const & t)
 {
-       for (int i = 0;  i <= 255; ++i) {
-               if (isalpha(i))
-                       lexcode[i] = LexAlpha;
-               else if (isdigit(i))
-                       lexcode[i] = LexDigit;
-               else if (isspace(i))
-                       lexcode[i] = LexSpace;
-               else
-                       lexcode[i] = LexNone;
-       }
-       
-       lexcode['\t'] = lexcode['\f'] = lexcode[' '] = LexSpace;
-       lexcode['\n'] = LexNewLine;
-       lexcode['%'] = LexComment;
-       lexcode['#'] = LexArgument;
-       lexcode['+'] = lexcode['-'] = lexcode['*'] = lexcode['/']
-               = lexcode['<'] = lexcode['>'] = lexcode['='] = LexBOP;
-       
-       lexcode['!'] = lexcode[','] = lexcode[':']
-               = lexcode[';'] = LexMathSpace;
-       
-       lexcode['('] = lexcode[')'] = lexcode['|'] = lexcode['.'] =
-               lexcode['?'] = LexOther; 
-       
-       lexcode['\''] = lexcode['@'] = LexAlpha;
-       
-       lexcode['['] = lexcode[']'] = lexcode['^'] = lexcode['_'] = 
-               lexcode['&'] = LexSelf;  
-       
-       lexcode['\\'] = LexESC;
-       lexcode['{'] = LexOpen;
-       lexcode['}'] = LexClose;
-}
-
-
-static
-char LexGetArg(char lf, bool accept_spaces = false)
-{
-       // unsigned char c;
-       // char cc;
-       while (yyis->good()) {
-               char cc;
-               yyis->get(cc);
-               unsigned char c = cc;
-               if (c > ' ') {
-                       if (!lf) 
-                               lf = c;
-                       else if (c != lf) {
-                               lyxerr << "Math parse error: unexpected '"
-                                      << c << "'" << endl;
-                               return '\0';
-                       }
+       tokens_.push_back(t);
+}
+
+
+void Parser::pop_back()
+{
+       tokens_.pop_back();
+}
+
+
+Token const & Parser::prevToken() const
+{
+       static const Token dummy;
+       return pos_ > 0 ? tokens_[pos_ - 1] : dummy;
+}
+
+
+Token const & Parser::nextToken() const
+{
+       static const Token dummy;
+       return good() ? tokens_[pos_] : dummy;
+}
+
+
+Token const & Parser::getToken()
+{
+       static const Token dummy;
+       return good() ? tokens_[pos_++] : dummy;
+}
+
+
+void Parser::putback()
+{
+       --pos_;
+}
+
+
+bool Parser::good() const
+{
+       return pos_ < tokens_.size();
+}
+
+
+char Parser::getChar()
+{
+       if (!good())
+               lyxerr << "The input stream is not well..." << endl;
+       return tokens_[pos_++].character();
+}
+
+
+string Parser::getArg(char lf, char rg)
+{
+       string result;
+       char c = getChar();
+
+       if (c != lf)  
+               putback();
+       else 
+               while ((c = getChar()) != rg && good())
+                       result += c;
+
+       return result;
+}
+
+
+void Parser::tokenize(istream & is)
+{
+       // eat everything up to the next \end_inset or end of stream
+       // and store it in s for further tokenization
+       string s;
+       char c;
+       while (is.get(c)) {
+               s += c;
+               if (s.size() >= 10 && s.substr(s.size() - 10) == "\\end_inset") {
+                       s = s.substr(0, s.size() - 10);
                        break;
                }
        }
-       char const rg =
-               (lf == '{') ? '}' :
-               ((lf == '[') ? ']'
-                : ((lf == '(') ? ')' : 0));
-       if (!rg) {
-               lyxerr << "Math parse error: unknown bracket '"
-                      << lf << "'" << endl;
-               return '\0';
-       }
-       char * p = &yytext[0];
-       int bcnt = 1;
-       do {
-               char cc;
-               yyis->get(cc);
-               unsigned char c = cc;
-               if (c == lf) ++bcnt;
-               if (c == rg) --bcnt;
-               if ((c > ' ' || (c == ' ' && accept_spaces)) && bcnt > 0)
-                       *(p++) = c;
-       } while (bcnt > 0 && yyis->good() && p - yytext.data() < 255);
-       
-       *p = '\0';
-       return rg;
+
+       // tokenize buffer
+       tokenize(s);
 }
 
 
-static
-int yylex(void)
+void Parser::tokenize(string const & buffer)
 {
-       static int init_done = 0;
-       
-       if (!init_done) LexInitCodes();
+       static bool init_done = false;
        
-       unsigned char c;
-       char cc;
-       while (yyis->good()) {
-               yyis->get(cc);
-               c = cc;
-               
-               if (yy_mtextmode && c == ' ') {
-                       yylval.i= ' ';
-                       return LM_TK_ALPHA;
-               } else if (lexcode[c] == LexNewLine) {
-                       ++yylineno; 
-                       continue;
-               } else if (lexcode[c] == LexComment) {
-                       do {
-                               yyis->get(cc);
-                               c = cc;
-                       } while (c != '\n' % yyis->good());  // eat comments
-               } else if (lexcode[c] == LexDigit
-                          || lexcode[c] == LexOther
-                          || lexcode[c] == LexMathSpace) {
-                       yylval.i = c;
-                       return LM_TK_STR;
-               } else if (lexcode[c] == LexAlpha) {
-                       yylval.i= c;
-                       return LM_TK_ALPHA;
-               } else if (lexcode[c] == LexBOP) {
-                       yylval.i= c;
-                       return LM_TK_BOP;
-               } else if (lexcode[c] == LexSelf) {
-                       return c;
-               } else if (lexcode[c] == LexArgument) {
-                       yyis->get(cc);
-                       c = cc;
-                       yylval.i = c - '0';
-                       return LM_TK_ARGUMENT; 
-               } else if (lexcode[c] == LexOpen) {
-                       return LM_TK_OPEN;
-               } else if (lexcode[c] == LexClose) {
-                       return LM_TK_CLOSE;
-               } else if (lexcode[c] == LexESC)   {
-                       yyis->get(cc);
-                       c = cc;
-                       if (c == '\\')  {
-                               return LM_TK_NEWLINE;
-                       }
-                       if (c == '(') {
-                               yylval.i = LM_OT_MIN;
-                               return LM_TK_BEGIN;
-                       }
-                       if (c == ')') {
-                               yylval.i = LM_OT_MIN;
-                               return LM_TK_END;
-                       }
-                       if (c == '[') {
-                               yylval.i = LM_OT_PAR;
-                               return LM_TK_BEGIN;
-                       }
-                       if (c == ']') {
-                               yylval.i = LM_OT_PAR;
-                               return LM_TK_END;
+       if (!init_done) {
+               catInit();
+               init_done = true;
+       }
+
+       istringstream is(buffer.c_str(), ios::in | ios::binary);
+
+       char c;
+       while (is.get(c)) {
+
+               switch (catcode(c)) {
+                       case catNewline: {
+                               ++lineno_; 
+                               is.get(c);
+                               if (catcode(c) == catNewline)
+                                       ; //push_back(Token("par"));
+                               else {
+                                       push_back(Token(' ', catSpace));
+                                       is.putback(c);  
+                               }
+                               break;
                        }
-                       if (strchr(latex_special_chars, c)) {
-                               yylval.i = c;
-                               return LM_TK_SPECIAL;
-                       } 
-                       if (lexcode[c] == LexMathSpace) {
-                               int i;
-                               for (i = 0; i < 4 && static_cast<int>(c) != latex_mathspace[i][0]; ++i);
-                               yylval.i = (i < 4) ? i : 0; 
-                               return LM_TK_SPACE; 
+
+                       case catComment: {
+                               while (is.get(c) && catcode(c) != catNewline)
+                                       ;
+                               ++lineno_; 
+                               break;
                        }
-                       if (lexcode[c] == LexAlpha || lexcode[c] == LexDigit) {
-                               char * p = &yytext[0];
-                               while ((lexcode[c] == LexAlpha || lexcode[c] == LexDigit)
-                                      && p - yytext.data() < 255) {
-                                       *p = c;
-                                       yyis->get(cc);
-                                       c = cc;
-                                       ++p;
-                               }
-                               *p = '\0';
-                               if (yyis->good()) yyis->putback(c);
-                               latexkeys const * l = in_word_set (yytext.data(), strlen(yytext.data()));
-                               if (l) {
-                                       if (l->token == LM_TK_BEGIN || l->token == LM_TK_END) { 
-                                               int i;
-                                               LexGetArg('{');
-//               for (i = 0; i < 5 && compare(yytext, latex_mathenv[i],
-//                             strlen(latex_mathenv[i])); ++i);
-                                               
-                                               for (i = 0;
-                                                    i < latex_mathenv_num
-                                                            && compare(yytext.data(), latex_mathenv[i]); ++i);
-                                               yylval.i = i;
-                                       } else if (l->token == LM_TK_SPACE) 
-                                               yylval.i = l->id;
-                                       else
-                                               yylval.l = l;
-                                       return l->token;
-                               } else { 
-                                       yylval.s = yytext.data();
-                                       return LM_TK_UNDEF;
-                               }
+
+                       case catEscape: {
+                               is.get(c);
+                               string s(1, c);
+                               if (catcode(c) == catLetter) {
+                                       while (is.get(c) && catcode(c) == catLetter)
+                                               s += c;
+                                       if (catcode(c) == catSpace)
+                                               while (is.get(c) && catcode(c) == catSpace)
+                                                       ;
+                                       is.putback(c);
+                               }       
+                               push_back(Token(s));
+                               break;
                        }
+
+                       default:
+                               push_back(Token(c, catcode(c)));
                }
        }
-       return 0;
+
+#if 0
+       lyxerr << "\nTokens: ";
+       for (unsigned i = 0; i < tokens_.size(); ++i)
+               lyxerr << tokens_[i];
+       lyxerr << "\n";
+#endif
 }
 
 
-static inline
-int parse_align(char * hor, char *)
+void Parser::error(string const & msg) 
 {
-       int nc = 0;
-       for (char * c = hor; c && *c > ' '; ++c) ++nc;
-       return nc;
+       lyxerr << "Line ~" << lineno_ << ": Math parse error: " << msg << endl;
+       //exit(1);
 }
 
 
-// Accent hacks only for 0.12. Stolen from Cursor.
-static
-int accent = 0;
-static
-int nestaccent[8];
-
-static inline
-void setAccent(int ac)
+void Parser::parse_lines(MathGridInset * p, bool numbered, bool outmost)
 {
-       if (ac > 0 && accent < 8) {
-               nestaccent[accent++] = ac;
-       } else
-         accent = 0;  // consumed!
+       const int cols = p->ncols();
+
+       // save global variables
+       bool   const saved_num   = curr_num_;
+       string const saved_label = curr_label_;
+
+       for (int row = 0; true; ++row) {
+               // reset global variables
+               curr_num_   = numbered;
+               curr_label_.erase();
+
+               // reading a row
+               for (int col = 0; col < cols; ++col) {
+                       //lyxerr << "reading cell " << row << " " << col << "\n";
+                       parse_into(p->cell(col + row * cols), FLAG_BLOCK);
+
+                       // no ampersand
+                       if (prevToken().cat() != catAlign) {
+                               //lyxerr << "less cells read than normal in row/col: "
+                               //      << row << " " << col << "\n";
+                               break;
+                       }
+               }
+
+               if (outmost) {
+                       MathMatrixInset * m = static_cast<MathMatrixInset *>(p);
+                       m->numbered(row, curr_num_);
+                       m->label(row, curr_label_);
+                       if (curr_skip_.size()) {
+                               m->vskip(LyXLength(curr_skip_), row);
+                               curr_skip_.erase();
+                       }
+               }
+
+               // no newline?
+               if (prevToken() != Token("\\")) {
+                       //lyxerr << "no newline here\n";
+                       break;
+               }
+
+               p->appendRow();
+       }
+
+       // restore "global" variables
+       curr_num_   = saved_num;
+       curr_label_ = saved_label;
 }
 
 
-static
-MathedInset * doAccent(byte c, MathedTextCodes t)
+MathMacroTemplate * Parser::parse_macro()
 {
-       MathedInset * ac = 0;
-       
-       for (int i = accent - 1; i >= 0; --i) {
-               if (i == accent - 1)
-                 ac = new MathAccentInset(c, t, nestaccent[i]);
-               else 
-                 ac = new MathAccentInset(ac, nestaccent[i]);
+       while (nextToken().cat() == catSpace)
+               getToken();
+
+       if (getToken().cs() != "newcommand") {
+               lyxerr << "\\newcommand expected\n";
+               return 0;
        }
-       accent = 0;  // consumed!
-       
-       return ac;
+
+       if (getToken().cat() != catBegin) {
+               lyxerr << "'{' expected\n";
+               return 0;
+       }
+
+       string name = getToken().cs();
+
+       if (getToken().cat() != catEnd) {
+               lyxerr << "'}' expected\n";
+               return 0;
+       }
+
+       string arg  = getArg('[', ']');
+       int    narg = arg.empty() ? 0 : atoi(arg.c_str()); 
+       //lyxerr << "creating macro " << name << " with " << narg <<  "args\n";
+       MathMacroTemplate * p = new MathMacroTemplate(name, narg);
+       parse_into(p->cell(0), FLAG_BRACE | FLAG_BRACE_LAST);
+       return p;
 }
 
 
-static
-MathedInset * doAccent(MathedInset * p)
+MathMatrixInset * Parser::parse_normal()
 {
-       MathedInset * ac = 0;
-       
-       for (int i = accent - 1; i >= 0; --i) {
-               if (i == accent - 1)
-                 ac = new MathAccentInset(p, nestaccent[i]);
-               else 
-                 ac = new MathAccentInset(ac, nestaccent[i]);
+       while (nextToken().cat() == catSpace)
+               getToken();
+
+       Token const & t = getToken();
+
+       if (t.cat() == catMath || t.cs() == "(") {
+               MathMatrixInset * p = new MathMatrixInset(LM_OT_SIMPLE);
+               parse_into(p->cell(0), 0);
+               return p;
        }
-       accent = 0;  // consumed!
-       
-       return ac;
+
+       if (!t.cs().size()) {
+               lyxerr << "start of math expected, got '" << t << "'\n";
+               return 0;
+       }
+
+       string const & cs = t.cs();
+
+       if (cs == "[") {
+               curr_num_ = 0;
+               curr_label_.erase();
+               MathMatrixInset * p = new MathMatrixInset(LM_OT_EQUATION);
+               parse_into(p->cell(0), 0);
+               p->numbered(0, curr_num_);
+               p->label(0, curr_label_);
+               return p;
+       }
+
+       if (cs != "begin") {
+               lyxerr << "'begin' of un-simple math expected, got '" << cs << "'\n";
+               return 0;
+       }
+
+       string const name = getArg('{', '}');
+
+       if (name == "equation" || name == "equation*") {
+               curr_num_ = !stared(name);
+               curr_label_.erase();
+               MathMatrixInset * p = new MathMatrixInset(LM_OT_EQUATION);
+               parse_into(p->cell(0), FLAG_END);
+               p->numbered(0, curr_num_);
+               p->label(0, curr_label_);
+               return p;
+       }
+
+       if (name == "eqnarray" || name == "eqnarray*") {
+               MathMatrixInset * p = new MathMatrixInset(LM_OT_EQNARRAY);
+               parse_lines(p, !stared(name), true);
+               return p;
+       }
+
+       if (name == "align" || name == "align*") {
+               MathMatrixInset * p = new MathMatrixInset(LM_OT_ALIGN);
+               parse_lines(p, !stared(name), true);
+               return p;
+       }
+
+       if (name == "alignat" || name == "alignat*") {
+               MathMatrixInset * p =
+                       new MathMatrixInset(LM_OT_ALIGNAT, 2 * atoi(getArg('{', '}').c_str()));
+               parse_lines(p, !stared(name), true);
+               return p;
+       }
+
+       if (name == "xalignat" || name == "xalignat*") {
+               MathMatrixInset * p =
+                       new MathMatrixInset(LM_OT_XALIGNAT, 2 * atoi(getArg('{', '}').c_str()));
+               parse_lines(p, !stared(name), true);
+               return p;
+       }
+
+       if (name == "xxalignat") {
+               MathMatrixInset * p =
+                       new MathMatrixInset(LM_OT_XXALIGNAT, 2 * atoi(getArg('{', '}').c_str()));
+               parse_lines(p, !stared(name), true);
+               return p;
+       }
+
+       if (name == "multline" || name == "multline*") {
+               MathMatrixInset * p = new MathMatrixInset(LM_OT_MULTLINE);
+               parse_lines(p, !stared(name), true);
+               return p;
+       }
+
+       if (name == "gather" || name == "gather*") {
+               MathMatrixInset * p = new MathMatrixInset(LM_OT_GATHER);
+               parse_lines(p, !stared(name), true);
+               return p;
+       }
+
+       lyxerr[Debug::MATHED] << "1: unknown math environment: " << name << "\n";
+       return 0;
 }
 
 
-/**
- */
-void mathed_parse(MathedArray & array, unsigned flags = 0,
-                           MathParInset ** mtx = 0)
-{
-       int t = yylex();
-       int tprev = 0;
-       bool panic = false;
-       static int plevel = -1;
-       static int size = LM_ST_TEXT;
-       MathedTextCodes varcode = LM_TC_VAR;
-       MathedInset * binset = 0;
-       static MathMacroTemplate * macro = 0;
+void Parser::parse_into(MathArray & array, unsigned flags, MathTextCodes code)
+{
+       MathTextCodes yyvarcode = LM_TC_MIN;
+
+       bool panic  = false;
+
+       while (good()) {
+               Token const & t = getToken();
        
-       int brace = 0;
-       int acc_brace = 0;
-       int acc_braces[8];
-       MathParInset * mt = (mtx) ? *mtx : 0;
-       MathedRowContainer::iterator crow;
-       if (mt)
-               crow = mt->getRowSt().begin();
-
-       ++plevel;
-       MathedIter data(&array);
-       while (t) {
-               if ((flags & FLAG_BRACE) && t != LM_TK_OPEN) {
-                       if ((flags & FLAG_BRACK_ARG) && t == '[') {
+               //lyxerr << "t: " << t << " flags: " << flags << "'\n";
+               //array.dump(lyxerr);
+               //lyxerr << "\n";
+
+               if (flags & FLAG_ITEM) {
+                       flags &= ~FLAG_ITEM;
+                       if (t.cat() == catBegin) { 
+                               // skip the brace and collect everything to the next matching
+                               // closing brace
+                               flags |= FLAG_BRACE_LAST;
+                               continue;
                        } else {
-                               mathPrintError("Expected {. Maybe you forgot to enclose an argument in {}");
-                               panic = true;
-                               break;
+                               // handle only this single token, leave the loop if done
+                               flags |= FLAG_LEAVE;
                        }
                }
-               MathedInsetTypes fractype = LM_OT_FRAC;
-               switch (t) {
-                       
-               case LM_TK_ALPHA:
-                       if (accent) {
-                               data.insertInset(doAccent(yylval.i, varcode),
-                                                LM_TC_INSET);
-                       } else
-                               data.insert(yylval.i, varcode);  //LM_TC_VAR);
-                       break;
 
-               case LM_TK_ARGUMENT:
-                       if (macro) {
-                               data.insertInset(macro
-                                                ->getMacroPar(yylval.i - 1),
-                                                LM_TC_INSET);
+               if (flags & FLAG_BRACE) {
+                       if (t.cat() != catBegin) {
+                               error("Expected {. Maybe you forgot to enclose an argument in {}");
+                               panic = true;
+                               break;
                        } else {
-                               lyxerr[Debug::MATHED] << "mathed_parse: macro arg outside macro def." << endl;
+                               flags &= ~FLAG_BRACE;
+                               continue;
                        }
-                       
-                       break;
+               }
 
-               case LM_TK_NEWCOMMAND:
-               {
-                       int na = 0; 
-                       
-                       LexGetArg('{');
-                       string const name(&yytext[1]);
-                       
-                       // ugly trick to be removed soon (lyx3)
-                       char const c = yyis->peek();
-                       if (c == '[') {
-                               LexGetArg('[');
-                               na = lyx::atoi(yytext.data());
-                       }  
-                       macro = new MathMacroTemplate(name, na);
-                       flags = FLAG_BRACE|FLAG_BRACE_LAST;
-
-                       *mtx = macro;
-                       macro->setData(array);
-                       break;
+               if (flags & FLAG_BLOCK) {
+                       if (t.cat() == catAlign || t.cs() == "\\")
+                               return;
+                       if (t.cs() == "end") {
+                               getArg('{', '}');
+                               return;
+                       }
                }
-               
-               case LM_TK_SPECIAL:
-                       data.insert(yylval.i, LM_TC_SPECIAL);
-                       break;
 
-               case LM_TK_STR:
-                       if (accent) {
-                               data.insertInset(doAccent(yylval.i, LM_TC_CONST), LM_TC_INSET);
-                       } else
-                               data.insert(yylval.i, LM_TC_CONST);
+               //
+               // cat codes
+               //
+               if (t.cat() == catMath)
                        break;
 
-               case LM_TK_OPEN:
-                       ++brace;
-                       if  (accent && tprev == LM_TK_ACCENT) {
-                               acc_braces[acc_brace++] = brace;
-                               break;
-                       }
-                       if (flags & FLAG_BRACE_OPT) {
-                               flags &= ~FLAG_BRACE_OPT;
-                               flags |= FLAG_BRACE;
-                       }
-                       
-                       if (flags & FLAG_BRACE)
-                               flags &= ~FLAG_BRACE;
-                       else {
-                               data.insert('{', LM_TC_TEX);
-                       }
-                       break;
+               else if (t.cat() == catLetter)
+                       array.push_back(new MathCharInset(t.character(), yyvarcode));
 
-               case LM_TK_CLOSE:
-                       --brace;         
-                       if (brace < 0) {
-                               mathPrintError("Unmatching braces");
-                               panic = true;
-                               break;
-                       }
-                       if (acc_brace && brace == acc_braces[acc_brace - 1] - 1) {
-                               --acc_brace;
-                               break;
-                       }
-                       if (flags & FLAG_BRACE_FONT) {
-                               varcode = LM_TC_VAR;
-                               yy_mtextmode = false;
-                               flags &= ~FLAG_BRACE_FONT;
-                               break;
-                       }
-                       if (brace == 0 && (flags & FLAG_BRACE_LAST)) {
-                               --plevel;
-                               return;
-                       } else {
-                               data.insert('}', LM_TC_TEX);
-                       }
-                       break;
-               
-               case '[':
-                       if (flags & FLAG_BRACK_ARG) {
-                               flags &= ~FLAG_BRACK_ARG;
-                               char const rg = LexGetArg('[');
-                               if (rg != ']') {
-                                       mathPrintError("Expected ']'");
-                                       panic = true;
-                                       break;
-                               }          
-                               // if (arg) strcpy(arg, yytext);
-                       } else
-                               data.insert('[', LM_TC_CONST);
-                       break;
+               else if (t.cat() == catSpace &&
+                               (yyvarcode == LM_TC_TEXTRM || code == LM_TC_TEXTRM))
+                       array.push_back(new MathCharInset(' ', yyvarcode));
+
+               else if (t.cat() == catParameter) {
+                       Token const & n = getToken();
+                       MathMacroArgument * p = new MathMacroArgument(n.character() - '0');
+                       array.push_back(p);
+               }
+
+               else if (t.cat() == catBegin) {
+                       array.push_back(new MathCharInset('{', LM_TC_TEX));
+               }
 
-               case ']':
-                       if (flags & FLAG_BRACK_END) {
-                               --plevel;
+               else if (t.cat() == catEnd) {
+                       if (flags & FLAG_BRACE_LAST)
                                return;
-                       } else
-                               data.insert(']', LM_TC_CONST);
-                       break;
-               
-               case '^':
-               {  
-                       MathParInset * p = new MathParInset(size, "",
-                                                           LM_OT_SCRIPT);
-                       MathedArray ar;
-                       mathed_parse(ar, FLAG_BRACE_OPT|FLAG_BRACE_LAST);
-                       p->setData(ar);
-                       // lyxerr << "UP[" << p->GetStyle() << "]" << endl;
-                       data.insertInset(p, LM_TC_UP);
-                       break;
+                       array.push_back(new MathCharInset('}', LM_TC_TEX));
                }
                
-               case '_':
-               {
-                       MathParInset * p = new MathParInset(size, "",
-                                                           LM_OT_SCRIPT);
-                       MathedArray ar;
-                       mathed_parse(ar, FLAG_BRACE_OPT|FLAG_BRACE_LAST);
-                       p->setData(ar);
-                       data.insertInset(p, LM_TC_DOWN);
-                       break;
+               else if (t.cat() == catAlign) {
+                       lyxerr << "found tab unexpectedly, array: '" << array << "'\n";
+                       array.push_back(new MathCharInset('&', LM_TC_TEX));
                }
                
-               case LM_TK_LIMIT:
-                       if (binset) {
-                               binset->SetLimits(bool(yylval.l->id));
-                               binset = 0;
-                       }
-                       break;
-               
-               case '&':    // Tab
-                       if ((flags & FLAG_END) && mt
-                           && data.getCol()<mt->GetColumns() - 1) {
-                               data.setNumCols(mt->GetColumns());
-                               data.insert('T', LM_TC_TAB);
-                       } else 
-                               mathPrintError("Unexpected tab");
-                       // debug info. [made that conditional -JMarc]
-                       if (lyxerr.debugging(Debug::MATHED))
-                               lyxerr << data.getCol() << " "
-                                      << mt->GetColumns() << endl;
-                       break;
-               
-               case LM_TK_NEWLINE:
-                       if (mt && (flags & FLAG_END)) {
-                               if (mt->Permit(LMPF_ALLOW_CR)) {
-                                       mt->getRowSt().insert_after(crow, MathedRowSt(mt->GetColumns() + 1));
-                                       ++crow;
-                                       data.insert('K', LM_TC_CR);
-                               } else 
-                                       mathPrintError("Unexpected newline");
-                       }
-                       break;
-
-               case LM_TK_BIGSYM:  
-               {
-                       binset = new MathBigopInset(yylval.l->name, yylval.l->id);
-                       data.insertInset(binset, LM_TC_INSET);  
-                       break;
+               else if (t.cat() == catSuper || t.cat() == catSub) {
+                       bool up = (t.cat() == catSuper);
+                       if (array.empty())
+                               array.push_back(new MathCharInset(' '));
+                       parse_into(array.back().ensure(up)->cell(0), FLAG_ITEM);
                }
+
+               else if (t.character() == ']' && (flags & FLAG_BRACK_END))
+                       return;
+
+               else if (t.cat() == catOther)
+                       array.push_back(new MathCharInset(t.character(), yyvarcode));
                
-               case LM_TK_SYM:
-                       if (yylval.l->id < 256) {
-                               MathedTextCodes tc = MathIsBOPS(yylval.l->id) ? LM_TC_BOPS: LM_TC_SYMB;
-                               if (accent) {
-                                       data.insertInset(doAccent(yylval.l->id, tc), LM_TC_INSET);
-                               } else
-                                       data.insert(yylval.l->id, tc);
-                       } else {
-                               MathFuncInset * bg = new MathFuncInset(yylval.l->name);
-                               if (accent) {
-                                       data.insertInset(doAccent(bg), LM_TC_INSET);
-                               } else {
-#warning This is suspisious! (Lgb)
-                                       // it should not take a bool as second arg (Lgb)
-                                       data.insertInset(bg, true);
-                               }
-                               
-                       }
-                       break;
+               //
+               // codesequences
+               //      
+               else if (t.cs() == "protect") 
+                       ;
 
-               case LM_TK_BOP:
-                       if (accent) {
-                               data.insertInset(doAccent(yylval.i, LM_TC_BOP), LM_TC_INSET);
-                       } else
-                               data.insert(yylval.i, LM_TC_BOP);
+               else if (t.cs() == "end")
                        break;
 
-               case LM_TK_STY:
-                       if (mt) {
-                               mt->UserSetSize(yylval.l->id);
-                       }
-                       break; 
-
-               case LM_TK_SPACE:
-                       if (yylval.i >= 0) {
-                               MathSpaceInset * sp = new MathSpaceInset(yylval.i);
-                               data.insertInset(sp, LM_TC_INSET);
-                       }
+               else if (t.cs() == ")")
                        break;
 
-               case LM_TK_DOTS:
-               {
-                       MathDotsInset * p = new MathDotsInset(yylval.l->name, yylval.l->id);
-                       data.insertInset(p, LM_TC_INSET);
+               else if (t.cs() == "]")
                        break;
+
+               else if (t.cs() == "\\") {
+                       curr_skip_ = getArg('[', ']');
+                       if (flags & FLAG_NEWLINE)
+                               return;
+                       lyxerr[Debug::MATHED]
+                                       << "found newline unexpectedly, array: '" << array << "'\n";
+                       array.push_back(createMathInset("\\"));
                }
+       
+               else if (t.cs() == "limits" && array.size())
+                       array.back().limits(1);
                
-               case LM_TK_STACK:
-                       fractype = LM_OT_STACKREL;
-                       // fallthru
-               case LM_TK_FRAC:
-               {
-                       MathFracInset * fc = new MathFracInset(fractype);
-                       MathedArray num;
-                       mathed_parse(num, FLAG_BRACE|FLAG_BRACE_LAST);
-                       MathedArray den;
-                       mathed_parse(den, FLAG_BRACE|FLAG_BRACE_LAST);
-                       fc->SetData(num, den);
-                       data.insertInset(fc, LM_TC_ACTIVE_INSET);
-                       break;
-               }
+               else if (t.cs() == "nolimits" && array.size())
+                       array.back().limits(-1);
                
-               case LM_TK_SQRT:
-               {           
-                       MathParInset * rt;
-                       
-                       char c;
-                       yyis->get(c);
-                       
+               else if (t.cs() == "nonumber")
+                       curr_num_ = false;
+
+               else if (t.cs() == "number")
+                       curr_num_ = true;
+
+               else if (t.cs() == "sqrt") {
+                       char c = getChar();
                        if (c == '[') {
-                               rt = new MathRootInset(size);
-                               rt->setArgumentIdx(0);
-                               MathedArray ar;
-                               mathed_parse(ar, FLAG_BRACK_END, &rt);
-                               rt->setData(ar); // I belive that line is not needed (Lgb)
-                               rt->setArgumentIdx(1);
+                               array.push_back(new MathRootInset);
+                               parse_into(array.back().nucleus()->cell(0), FLAG_BRACK_END);
+                               parse_into(array.back().nucleus()->cell(1), FLAG_ITEM);
                        } else {
-                               yyis->putback(c);
-                               rt = new MathSqrtInset(size);
+                               putback();
+                               array.push_back(new MathSqrtInset);
+                               parse_into(array.back().nucleus()->cell(0), FLAG_ITEM);
                        }
-                       MathedArray ar;
-                       mathed_parse(ar, FLAG_BRACE|FLAG_BRACE_LAST, &rt);
-                       rt->setData(ar); // I belive that this line is not needed (Lgb)
-                       data.insertInset(rt, LM_TC_ACTIVE_INSET);
-                       break;
                }
                
-               case LM_TK_LEFT:
-               {
-                       int lfd = yylex();
-                       if (lfd == LM_TK_SYM || lfd == LM_TK_STR || lfd == LM_TK_BOP|| lfd == LM_TK_SPECIAL)
-                               lfd = (lfd == LM_TK_SYM) ? yylval.l->id: yylval.i;
-//      lyxerr << "L[" << lfd << " " << lfd << "]";
-                       MathedArray ar;
-                       mathed_parse(ar, FLAG_RIGHT);
-                       int rgd = yylex();
-//      lyxerr << "R[" << rgd << "]";
-                       if (rgd == LM_TK_SYM || rgd == LM_TK_STR || rgd == LM_TK_BOP || rgd == LM_TK_SPECIAL)
-                               rgd = (rgd == LM_TK_SYM) ? yylval.l->id: yylval.i;       
-                       MathDelimInset * dl = new MathDelimInset(lfd, rgd);
-                       dl->setData(ar);
-                       data.insertInset(dl, LM_TC_ACTIVE_INSET);
-//      lyxerr << "RL[" << lfd << " " << rgd << "]";
-                       break;
+               else if (t.cs() == "left") {
+                       string l = getToken().asString();
+                       MathArray ar;
+                       parse_into(ar, FLAG_RIGHT);
+                       string r = getToken().asString();
+                       MathDelimInset * dl = new MathDelimInset(l, r);
+                       dl->cell(0) = ar;
+                       array.push_back(dl);
                }
                
-               case LM_TK_RIGHT:
-                       if (flags & FLAG_RIGHT) { 
-                               --plevel;
-                               return;
-                       } else {
-                               mathPrintError("Unmatched right delimiter");
-//         panic = true;
+               else if (t.cs() == "right") {
+                       if (!(flags & FLAG_RIGHT)) {
+                               lyxerr << "got so far: '" << array << "'\n";
+                               error("Unmatched right delimiter");
                        }
-                       break;
-               
-               case LM_TK_FONT:
-                       varcode = static_cast<MathedTextCodes>(yylval.l->id);
-                       yy_mtextmode = bool(varcode == LM_TC_TEXTRM);
-                       flags |= (FLAG_BRACE|FLAG_BRACE_FONT);
-                       break;
-
-               case LM_TK_WIDE:
-               {  
-                       MathDecorationInset * sq = new MathDecorationInset(yylval.l->id,
-                                                                          size);
-                       MathedArray ar;
-                       mathed_parse(ar, FLAG_BRACE|FLAG_BRACE_LAST);
-                       sq->setData(ar);
-                       data.insertInset(sq, LM_TC_ACTIVE_INSET);
-                       break;
+                       return;
                }
-               
-               case LM_TK_ACCENT:
-                       setAccent(yylval.l->id);
-                       break;
-                       
-               case LM_TK_NONUM:
-                       if (crow)
-                               crow->setNumbered(false);
-                       break;
-               
-               case LM_TK_PMOD:
-               case LM_TK_FUNC:
-                       if (accent) {
-                               data.insert(t, LM_TC_CONST);
-                       } else {
-                               MathedInset * bg = new MathFuncInset(yylval.l->name); 
-                               data.insertInset(bg, LM_TC_INSET);
-                       }
-                       break;
-               
-               case LM_TK_FUNCLIM:
-                       data.insertInset(new MathFuncInset(yylval.l->name, LM_OT_FUNCLIM),
-                                        LM_TC_INSET);
-                       break;
 
-               case LM_TK_UNDEF:
+/*             
+               case LM_TK_STY:
                {
-                       
-                       MathMacro * p = 
-                               MathMacroTable::mathMTable.createMacro(yylval.s);
-                       if (p) {
-                               if (accent) 
-                                       data.insertInset(doAccent(p), p->getTCode());
-                               else
-                                       data.insertInset(p, p->getTCode());
-                               for (int i = 0; p->setArgumentIdx(i); ++i) {
-                                       MathedArray ar;
-                                       mathed_parse(ar, FLAG_BRACE|FLAG_BRACE_LAST);
-                                       p->setData(ar);
-                               }
-                       } else {
-                               MathedInset * q = new MathFuncInset(yylval.s, LM_OT_UNDEF);
-                               if (accent) {
-                                       data.insertInset(doAccent(q), LM_TC_INSET);
-                               } else {
-                                       data.insertInset(q, LM_TC_INSET);
-                               }
-                       }
-                       break;
+                       lyxerr[Debug::MATHED] << "LM_TK_STY not implemented\n";
+                       //MathArray tmp = array;
+                       //MathSizeInset * p = new MathSizeInset(MathStyles(lval_->id));
+                       //array.push_back(p);
+                       //parse_into(p->cell(0), FLAG_BRACE_FONT);
+                       break; 
                }
+
+*/
                
-               case LM_TK_END:
-                       if (mathed_env != yylval.i && yylval.i != LM_OT_MATRIX)
-                               mathPrintError("Unmatched environment");
-                       // debug info [made that conditional -JMarc]
-                       if (lyxerr.debugging(Debug::MATHED))
-                               lyxerr << "[" << yylval.i << "]" << endl;
-                       --plevel;
-                       if (mt) { // && (flags & FLAG_END)) {
-                               mt->setData(array);
-                               array.clear();
+               else if (t.cs() == "begin") {
+                       string const name = getArg('{', '}');   
+                       if (name == "array") {
+                               string const valign = getArg('[', ']') + 'c';
+                               string const halign = getArg('{', '}');
+                               MathArrayInset * m = new MathArrayInset(halign.size(), 1);
+                               m->valign(valign[0]);
+                               m->halign(halign);
+                               parse_lines(m, false, false);
+                               array.push_back(m);
+                       } else if (name == "split") {
+                               MathSplitInset * m = new MathSplitInset(1);
+                               parse_lines(m, false, false);
+                               array.push_back(m);
+                       } else 
+                               lyxerr[Debug::MATHED] << "unknow math inset begin '" << name << "'\n";  
+               }
+       
+               else if (t.cs() == "kern") {
+#ifdef WITH_WARNINGS
+#warning A hack...
+#endif
+                       string s;
+                       while (1) {
+                               Token const & t = getToken();
+                               if (!good()) {
+                                       putback();      
+                                       break;
+                               }
+                               s += t.character();
+                               if (isValidLength(s))
+                                       break;
                        }
-                       return;
+                       array.push_back(new MathKernInset(s));
+               }
 
-               case LM_TK_BEGIN:
-                       if (yylval.i == LM_OT_MATRIX) {
-                               char ar[120];
-                               char ar2[8];
-                               ar[0] = ar2[0] = '\0'; 
-                               char rg = LexGetArg(0);
-                               if (rg == ']') {
-                                       strcpy(ar2, yytext.data());
-                                       rg = LexGetArg('{');
-                               }
-                               strcpy(ar, yytext.data());
-                               int const nc = parse_align(ar, ar2);
-                               MathParInset * mm = new MathMatrixInset(nc, 0);
-                               mm->SetAlign(ar2[0], ar);
-                               data.insertInset(mm, LM_TC_ACTIVE_INSET);
-                               MathedArray dat;
-                               mathed_parse(dat, FLAG_END, &mm);
-                       } else if (is_eqn_type(yylval.i)) {
-                               if (plevel!= 0) {
-                                       mathPrintError("Misplaced environment");
+               else if (t.cs() == "label") {
+                       //MathArray ar;
+                       //parse_into(ar, FLAG_ITEM);
+                       //ostringstream os;
+                       //ar.write(os, true);
+                       //curr_label_ = os.str();
+                       // was: 
+                       curr_label_ = getArg('{', '}');
+               }
+
+               else if (t.cs() == "choose" || t.cs() == "over" || t.cs() == "atop") {
+                       MathInset * p = createMathInset(t.cs());
+                       // search backward for position of last '{' if any
+                       int pos;
+                       for (pos = array.size() - 1; pos >= 0; --pos)
+                               if (array.at(pos)->nucleus()->getChar() == '{')
                                        break;
-                               }
-                               if (!mt) {
-                                       mathPrintError("0 paragraph.");
-                                       panic = true;
-                               }
-                               
-                               mathed_env = static_cast<MathedInsetTypes>(yylval.i);
-                               if (mathed_env != LM_OT_MIN) {
-                                       size = LM_ST_DISPLAY;
-                                       if (is_multiline(mathed_env)) {
-                                               int cols = 1;
-                                               if (is_multicolumn(mathed_env)) {
-                                                       if (mathed_env != LM_OT_ALIGNAT &&
-                                                           mathed_env != LM_OT_ALIGNATN &&
-                                                           yyis->good()) {
-                                                               char c;
-                                                               yyis->get(c);
-                                                               if (c != '%')
-                                                                       lyxerr << "Math parse error: unexpected '"
-                                                                              << c << "'" << endl;
-                                                       }
-                                                       LexGetArg('{');
-                                                       cols = strToInt(string(yytext.data()));
-                                               }
-                                               mt = create_multiline(mathed_env, cols);
-                                               if (mtx) *mtx = mt;
-                                               flags |= FLAG_END;
-//                  data.Insert(' ', LM_TC_TAB);
-//                  data.Insert(' ', LM_TC_TAB);
-//                  data.Reset();
-                                       }
-                                       mt->SetStyle(size);
-                                       mt->SetType(mathed_env);
-                                       crow = mt->getRowSt().begin();
-                               }
-                               
-                               lyxerr[Debug::MATHED] << "MATH BEGIN[" << mathed_env << "]" << endl;
+                       if (pos >= 0) {
+                               // found it -> use the part after '{' as "numerator"
+                               p->cell(0) = MathArray(array, pos + 1, array.size());
+                               parse_into(p->cell(1), FLAG_BRACE_LAST);
+                               // delete denominator and the '{'
+                               array.erase(pos, array.size());
+                       } else if (flags & FLAG_RIGHT) {
+                               // we are inside a \left ... \right block
+                               //lyxerr << "found '" << t.cs() << "' enclosed by \\left .. \\right\n";
+                               p->cell(0).swap(array);
+                               parse_into(p->cell(1), FLAG_RIGHT);
+                               // handle the right delimiter properly
+                               putback();
                        } else {
-//          lyxerr << "MATHCRO[" << yytext << "]";
-                               MathMacro * p = 
-                                       MathMacroTable::mathMTable.createMacro(yytext.data());
-                               if (p) {
-                                       data.insertInset(p, p->getTCode());
-                                       p->setArgumentIdx(0);
-                                       //mathed_parse(p->GetData(), FLAG_END, reinterpret_cast<MathParInset**>(&p));
-                                       MathedArray dat;
-                                       mathed_parse(dat, FLAG_END, reinterpret_cast<MathParInset**>(&p));
-//              for (int i = 0; p->setArgumentIdx(i); ++i)
-//                p->SetData(mathed_parse(FLAG_BRACE|FLAG_BRACE_LAST));
-                               } else 
-                                       mathPrintError("Unrecognized environment");
+                               // not found -> use everything as "numerator"
+                               p->cell(0).swap(array);
+                               parse_into(p->cell(1), FLAG_BLOCK);
                        }
-                       break;
-               
-               case LM_TK_MACRO:
-               { 
-                       MathedInset * p = 
-                               MathMacroTable::mathMTable.createMacro(yylval.l->name);
-                       
-                       if (p) {
-                               if (accent) {
-                                       data.insertInset(doAccent(p), LM_TC_INSET);
-                               } else
-                                       data.insertInset(p, static_cast<MathMacro*>(p)->getTCode());
-                       }
-                       break;
+                       array.push_back(p);
                }
-               
-               case LM_TK_LABEL:
-               {          
-                       char const rg = LexGetArg('\0', true);
-                       if (rg != '}') {
-                               mathPrintError("Expected '{'");
-                               // debug info
-                               lyxerr << "[" << yytext.data() << "]" << endl;
-                               panic = true;
-                               break;
-                       } 
-                       if (crow) {
-                               crow->setLabel(yytext.data());
-                       } else {
-                               mathed_label = yytext.data();
+       
+               else if (t.cs().size()) {
+                       latexkeys const * l = in_word_set(t.cs());
+                       if (l) {
+                               if (l->token == LM_TK_FONT) {
+                                       //lyxerr << "starting font\n";
+                                       //CatCode catSpaceSave = theCatcode[' '];
+                                       //if (l->id == LM_TC_TEXTRM) {
+                                       //      // temporarily change catcode   
+                                       //      theCatcode[' '] = catLetter;    
+                                       //}
+
+                                       MathTextCodes t = static_cast<MathTextCodes>(l->id);
+                                       MathArray ar;
+                                       parse_into(ar, FLAG_ITEM, t);
+                                       for (MathArray::iterator it = ar.begin(); it != ar.end(); ++it)
+                                               it->nucleus()->handleFont(t);
+                                       array.push_back(ar);
+
+                                       // undo catcode changes
+                                       ////theCatcode[' '] = catSpaceSave;
+                                       //lyxerr << "ending font\n";
+                               }
+
+                               else if (l->token == LM_TK_OLDFONT)
+                                       yyvarcode = static_cast<MathTextCodes>(l->id);
+
+                               else {
+                                       MathInset * p = createMathInset(t.cs());
+                                       for (unsigned int i = 0; i < p->nargs(); ++i) 
+                                               parse_into(p->cell(i), FLAG_ITEM);
+                                       array.push_back(p);
+                               }
+                       }
+
+                       else {
+                               MathInset * p = createMathInset(t.cs());
+                               if (p) {
+                                       for (unsigned int i = 0; i < p->nargs(); ++i)
+                                               parse_into(p->cell(i), FLAG_ITEM);
+                                       array.push_back(p);
+                               } else {
+                                       error("Unrecognized token");
+                                       //lyxerr[Debug::MATHED] << "[" << t << "]\n";
+                                       lyxerr << t << "\n";
+                               }       
                        }
-                       lyxerr[Debug::MATHED] << "Label[" << mathed_label << "]" << endl;
-                       break;
                }
-               
-               default:
-                       mathPrintError("Unrecognized token");
-                       // debug info
-                       lyxerr << "[" << t << " " << yytext.data() << "]" << endl;
-                       break;
-               } // end of switch
-               
-               tprev = t;
-               if (panic) {
-                       lyxerr << " Math Panic, expect problems!" << endl;
-                       //   Search for the end command. 
-                       do {
-                               t = yylex ();
-                       } while (t != LM_TK_END && t);
-               } else
-                       t = yylex ();
-               
-               if ((flags & FLAG_BRACE_OPT)/* && t!= '^' && t!= '_'*/) {
-                       flags &= ~FLAG_BRACE_OPT;
-                       //data.Insert (LM_TC_CLOSE);
+
+
+               if (flags & FLAG_LEAVE) {
+                       flags &= ~FLAG_LEAVE;
                        break;
                }
        }
-       --plevel;
+
+       if (panic) {
+               lyxerr << " Math Panic, expect problems!\n";
+               //   Search for the end command. 
+               Token t;
+               do {
+                       t = getToken();
+               } while (good() && t.cs() != "end");
+       }
+}
+
+} // anonymous namespace
+
+
+
+MathArray mathed_parse_cell(string const & str)
+{
+       istringstream is(str.c_str());
+       Parser parser(is);
+       MathArray ar;
+       parser.parse_into(ar, 0);
+       return ar;
+}
+
+
+
+MathMacroTemplate * mathed_parse_macro(string const & str)
+{
+       istringstream is(str.c_str());
+       Parser parser(is);
+       return parser.parse_macro();
 }
 
+MathMacroTemplate * mathed_parse_macro(istream & is)
+{
+       Parser parser(is);
+       return parser.parse_macro();
+}
 
-void mathed_parser_file(istream & is, int lineno)
+MathMacroTemplate * mathed_parse_macro(LyXLex & lex)
 {
-       yyis = &is;
-       yylineno = lineno;
-       if (!MathMacroTable::built)
-               MathMacroTable::mathMTable.builtinMacros();
+       Parser parser(lex);
+       return parser.parse_macro();
 }
 
 
-int mathed_parser_lineno()
+
+MathMatrixInset * mathed_parse_normal(string const & str)
+{
+       istringstream is(str.c_str());
+       Parser parser(is);
+       return parser.parse_normal();
+}
+
+MathMatrixInset * mathed_parse_normal(istream & is)
+{
+       Parser parser(is);
+       return parser.parse_normal();
+}
+
+MathMatrixInset * mathed_parse_normal(LyXLex & lex)
 {
-       return yylineno;
+       Parser parser(lex);
+       return parser.parse_normal();
 }