]> git.lyx.org Git - lyx.git/blobdiff - src/tex2lyx/text.cpp
tex2lyx/preamble.cpp: support Vietnamese (since format 291)
[lyx.git] / src / tex2lyx / text.cpp
index 227486e22c604b5f97c6db69db1f63c35c4fb51c..7eddc0ec80a01b98ca2cb66cb7504222ad0375fc 100644 (file)
@@ -3,9 +3,9 @@
  * This file is part of LyX, the document processor.
  * Licence details can be found in the file COPYING.
  *
- * \author André Pönitz
+ * \author André Pönitz
  * \author Jean-Marc Lasgouttes
- * \author Uwe Stöhr
+ * \author Uwe Stöhr
  *
  * Full author contact details are available in file CREDITS.
  */
 #include "tex2lyx.h"
 
 #include "Context.h"
+#include "Encoding.h"
 #include "FloatList.h"
 #include "Layout.h"
 #include "Length.h"
 
+#include "support/lassert.h"
 #include "support/convert.h"
 #include "support/FileName.h"
 #include "support/filetools.h"
 #include "support/lstrings.h"
 
+#include <algorithm>
 #include <iostream>
 #include <map>
 #include <sstream>
@@ -54,8 +57,8 @@ void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
                Context & context)
 {
        Context newcontext(context);
-       // Don't inherit the extra stuff
-       newcontext.extra_stuff.clear();
+       // Don't inherit the paragraph-level extra stuff
+       newcontext.par_extra_stuff.clear();
        parse_text(p, os, flags, outer, newcontext);
        // Make sure that we don't create invalid .lyx files
        context.need_layout = newcontext.need_layout;
@@ -80,22 +83,20 @@ string parse_text_snippet(Parser & p, unsigned flags, const bool outer,
        newcontext.need_end_layout = false;
        newcontext.new_layout_allowed = false;
        // Avoid warning by Context::~Context()
-       newcontext.extra_stuff.clear();
+       newcontext.par_extra_stuff.clear();
        ostringstream os;
        parse_text_snippet(p, os, flags, outer, newcontext);
        return os.str();
 }
 
 
-char const * const known_latex_commands[] = { "ref", "cite", "nocite", "label",
- "index", "printindex", "pageref", "url", "vref", "vpageref", "prettyref",
- "eqref", 0 };
+char const * const known_ref_commands[] = { "ref", "pageref", "vref",
+ "vpageref", "prettyref", "eqref", 0 };
 
 /*!
  * natbib commands.
- * We can't put these into known_latex_commands because the argument order
- * is reversed in lyx if there are 2 arguments.
- * The starred forms are also known.
+ * The starred forms are also known except for "citefullauthor",
+ * "citeyear" and "citeyearpar".
  */
 char const * const known_natbib_commands[] = { "cite", "citet", "citep",
 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
@@ -103,8 +104,6 @@ char const * const known_natbib_commands[] = { "cite", "citet", "citep",
 
 /*!
  * jurabib commands.
- * We can't put these into known_latex_commands because the argument order
- * is reversed in lyx if there are 2 arguments.
  * No starred form other than "cite*" known.
  */
 char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
@@ -113,7 +112,7 @@ char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
 // "fullcite",
 // "footcite", "footcitet", "footcitep", "footcitealt", "footcitealp",
 // "footciteauthor", "footciteyear", "footciteyearpar",
-"citefield", "citetitle", "cite*", 0 };
+"citefield", "citetitle", 0 };
 
 /// LaTeX names for quotes
 char const * const known_quotes[] = { "dq", "guillemotleft", "flqq", "og",
@@ -131,9 +130,9 @@ char const * const known_coded_quotes[] = { "prd", "ard", "ard", "ard",
 char const * const known_sizes[] = { "tiny", "scriptsize", "footnotesize",
 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
 
-/// the same as known_sizes with .lyx names
+/// the same as known_sizes with .lyx names plus a default entry
 char const * const known_coded_sizes[] = { "default", "tiny", "scriptsize", "footnotesize",
-"small", "normal", "large", "larger", "largest",  "huge", "giant", 0};
+"small", "normal", "large", "larger", "largest", "huge", "giant", 0};
 
 /// LaTeX 2.09 names for font families
 char const * const known_old_font_families[] = { "rm", "sf", "tt", 0};
@@ -193,27 +192,30 @@ char const * const known_tex_extensions[] = {"tex", 0};
 
 /// spaces known by InsetSpace
 char const * const known_spaces[] = { " ", "space", ",", "thinspace", "quad",
-"qquad", "enspace", "enskip", "negthinspace", 0};
+"qquad", "enspace", "enskip", "negthinspace", "hfill", "dotfill", "hrulefill",
+"leftarrowfill", "rightarrowfill", "upbracefill", "downbracefill", 0};
 
 /// the same as known_spaces with .lyx names
 char const * const known_coded_spaces[] = { "space{}", "space{}",
 "thinspace{}", "thinspace{}", "quad{}", "qquad{}", "enspace{}", "enskip{}",
-"negthinspace{}", 0};
+"negthinspace{}", "hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}",
+"rightarrowfill{}", "upbracefill{}", "downbracefill{}", 0};
 
 
-/// splits "x=z, y=b" into a map
-map<string, string> split_map(string const & s)
+/// splits "x=z, y=b" into a map and an ordered keyword vector
+void split_map(string const & s, map<string, string> & res, vector<string> & keys)
 {
-       map<string, string> res;
        vector<string> v;
        split(s, v);
+       res.clear();
+       keys.resize(v.size());
        for (size_t i = 0; i < v.size(); ++i) {
                size_t const pos   = v[i].find('=');
-               string const index = v[i].substr(0, pos);
-               string const value = v[i].substr(pos + 1, string::npos);
-               res[trim(index)] = trim(value);
+               string const index = trim(v[i].substr(0, pos));
+               string const value = trim(v[i].substr(pos + 1, string::npos));
+               res[index] = value;
+               keys[i] = index;
        }
-       return res;
 }
 
 
@@ -254,7 +256,7 @@ bool splitLatexLength(string const & len, string & value, string & unit)
 }
 
 
-/// A simple function to translate a latex length to something lyx can
+/// A simple function to translate a latex length to something LyX can
 /// understand. Not perfect, but rather best-effort.
 bool translate_len(string const & length, string & valstring, string & unit)
 {
@@ -342,8 +344,6 @@ void translate_box_len(string const & length, string & value, string & unit, str
 string find_file(string const & name, string const & path,
                 char const * const * extensions)
 {
-       // FIXME UNICODE encoding of name and path may be wrong (makeAbsPath
-       // expects utf8)
        for (char const * const * what = extensions; *what; ++what) {
                string const trial = addExtension(name, *what);
                if (makeAbsPath(trial, path).exists())
@@ -359,22 +359,31 @@ void begin_inset(ostream & os, string const & name)
 }
 
 
+void begin_command_inset(ostream & os, string const & name,
+                         string const & latexname)
+{
+       begin_inset(os, "CommandInset ");
+       os << name << "\nLatexCommand " << latexname << '\n';
+}
+
+
 void end_inset(ostream & os)
 {
        os << "\n\\end_inset\n\n";
 }
 
 
-void skip_braces(Parser & p)
+bool skip_braces(Parser & p)
 {
        if (p.next_token().cat() != catBegin)
-               return;
+               return false;
        p.get_token();
        if (p.next_token().cat() == catEnd) {
                p.get_token();
-               return;
+               return true;
        }
        p.putback();
+       return false;
 }
 
 
@@ -421,21 +430,50 @@ void handle_comment(ostream & os, string const & s, Context & context)
 }
 
 
-LayoutPtr findLayout(TextClass const & textclass, string const & name)
+Layout const * findLayout(TextClass const & textclass, string const & name)
 {
-       for (size_t i = 0; i != textclass.layoutCount(); ++i)
-               if (textclass.layout(i)->latexname() == name)
-                       return textclass.layout(i);
-       return LayoutPtr();
+       DocumentClass::const_iterator lit = textclass.begin();
+       DocumentClass::const_iterator len = textclass.end();
+       for (; lit != len; ++lit)
+               if (lit->latexname() == name)
+                       return &*lit;
+       return 0;
 }
 
 
 void eat_whitespace(Parser &, ostream &, Context &, bool);
 
 
+/*!
+ * Skips whitespace and braces.
+ * This should be called after a command has been parsed that is not put into
+ * ERT, and where LyX adds "{}" if needed.
+ */
+void skip_spaces_braces(Parser & p)
+{
+       /* The following four examples produce the same typeset output and
+          should be handled by this function:
+          - abc \j{} xyz
+          - abc \j {} xyz
+          - abc \j 
+            {} xyz
+          - abc \j %comment
+            {} xyz
+        */
+       // Unfortunately we need to skip comments, too.
+       // We can't use eat_whitespace since writing them after the {}
+       // results in different output in some cases.
+       bool const skipped_spaces = p.skip_spaces(true);
+       bool const skipped_braces = skip_braces(p);
+       if (skipped_spaces && !skipped_braces)
+               // put back the space (it is better handled by check_space)
+               p.unskip_spaces(true);
+}
+
+
 void output_command_layout(ostream & os, Parser & p, bool outer,
                           Context & parent_context,
-                          LayoutPtr newlayout)
+                          Layout const * newlayout)
 {
        parent_context.check_end_layout(os);
        Context context(true, parent_context.textclass, newlayout,
@@ -448,17 +486,37 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
        }
        context.check_deeper(os);
        context.check_layout(os);
-       if (context.layout->optionalargs > 0) {
+       unsigned int optargs = 0;
+       while (optargs < context.layout->optargs) {
                eat_whitespace(p, os, context, false);
-               if (p.next_token().character() == '[') {
-                       p.get_token(); // eat '['
-                       begin_inset(os, "OptArg\n");
-                       os << "status collapsed\n\n";
-                       parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
-                       end_inset(os);
-                       eat_whitespace(p, os, context, false);
-               }
+               if (p.next_token().character() != '[') 
+                       break;
+               p.get_token(); // eat '['
+               begin_inset(os, "OptArg\n");
+               os << "status collapsed\n\n";
+               parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
+               end_inset(os);
+               eat_whitespace(p, os, context, false);
+               ++optargs;
        }
+#if 0
+       // This is the code needed to parse required arguments, but 
+       // required arguments come into being only much later than the
+       // file format tex2lyx is presently outputting.
+       unsigned int reqargs = 0;
+       while (reqargs < context.layout->reqargs) {
+               eat_whitespace(p, os, context, false);
+               if (p.next_token().character() != '{') 
+                       break;
+               p.get_token(); // eat '{'
+               begin_inset(os, "OptArg\n");
+               os << "status collapsed\n\n";
+               parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
+               end_inset(os);
+               eat_whitespace(p, os, context, false);
+               ++reqargs;
+       }
+#endif
        parse_text(p, os, FLAG_ITEM, outer, context);
        context.check_end_layout(os);
        if (parent_context.deeper_paragraph) {
@@ -491,7 +549,7 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
  * The drawback is that the logic inside the function becomes
  * complicated, and that is the reason why it is not implemented.
  */
-void check_space(Parser const & p, ostream & os, Context & context)
+void check_space(Parser & p, ostream & os, Context & context)
 {
        Token const next = p.next_token();
        Token const curr = p.curr_token();
@@ -532,7 +590,8 @@ void parse_arguments(string const & command,
                        ert += '{' + p.verbatim_item() + '}';
                        break;
                case optional:
-                       ert += p.getOpt();
+                       // true because we must not eat whitespace
+                       ert += p.getOpt(true);
                        break;
                }
        }
@@ -639,7 +698,7 @@ void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
                parse_text_in_inset(p, os, flags, outer, parent_context);
                end_inset(os);
 #ifdef PRESERVE_LAYOUT
-               // lyx puts a % after the end of the minipage
+               // LyX puts a % after the end of the minipage
                if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) {
                        // new paragraph
                        //handle_comment(os, "%dummy", parent_context);
@@ -653,7 +712,8 @@ void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.skip_spaces();
                        // We add a protected space if something real follows
                        if (p.good() && p.next_token().cat() != catComment) {
-                               os << "\\InsetSpace ~\n";
+                               begin_inset(os, "Space ~\n");
+                               end_inset(os);
                        }
                }
 #endif
@@ -689,9 +749,9 @@ void parse_unknown_environment(Parser & p, string const & name, ostream & os,
 
 
 void parse_environment(Parser & p, ostream & os, bool outer,
-                      Context & parent_context)
+                       string & last_env, Context & parent_context)
 {
-       LayoutPtr newlayout;
+       Layout const * newlayout;
        string const name = p.getArg('{', '}');
        const bool is_starred = suffixIs(name, '*');
        string const unstarred_name = rtrim(name, "*");
@@ -747,6 +807,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
                end_inset(os);
                p.skip_spaces();
+               skip_braces(p); // eat {} that might by set by LyX behind comments
        }
 
        else if (name == "lyxgreyedout") {
@@ -759,21 +820,23 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                p.skip_spaces();
        }
 
-       else if (name == "framed") {
-               eat_whitespace(p, os, parent_context, false);
-               parent_context.check_layout(os);
-               begin_inset(os, "Note Framed\n");
-               os << "status open\n";
-               parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
-               end_inset(os);
-               p.skip_spaces();
-       }
-
-       else if (name == "shaded") {
+       else if (name == "framed" || name == "shaded") {
                eat_whitespace(p, os, parent_context, false);
                parent_context.check_layout(os);
-               begin_inset(os, "Note Shaded\n");
-               os << "status open\n";
+               if (name == "framed")
+                       begin_inset(os, "Box Framed\n");
+               else
+                       begin_inset(os, "Box Shaded\n");
+               os << "position \"t\"\n"
+                     "hor_pos \"c\"\n"
+                     "has_inner_box 0\n"
+                     "inner_pos \"t\"\n"
+                     "use_parbox 0\n"
+                     "width \"100col%\"\n"
+                     "special \"none\"\n"
+                     "height \"1in\"\n"
+                     "height_special \"totalheight\"\n"
+                     "status open\n";
                parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
                end_inset(os);
                p.skip_spaces();
@@ -783,23 +846,39 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                parse_unknown_environment(p, name, os, FLAG_END, outer,
                                          parent_context);
 
-       // Alignment settings
-       else if (name == "center" || name == "flushleft" || name == "flushright" ||
-                name == "centering" || name == "raggedright" || name == "raggedleft") {
+       // Alignment and spacing settings
+       // FIXME (bug xxxx): These settings can span multiple paragraphs and
+       //                                       therefore are totally broken!
+       // Note that \centering, raggedright, and raggedleft cannot be handled, as
+       // they are commands not environments. They are furthermore switches that
+       // can be ended by another switches, but also by commands like \footnote or
+       // \parbox. So the only safe way is to leave them untouched.
+       else if (name == "center" || name == "centering" ||
+                name == "flushleft" || name == "flushright" ||
+                name == "singlespace" || name == "onehalfspace" ||
+                name == "doublespace" || name == "spacing") {
                eat_whitespace(p, os, parent_context, false);
                // We must begin a new paragraph if not already done
                if (! parent_context.atParagraphStart()) {
                        parent_context.check_end_layout(os);
                        parent_context.new_paragraph(os);
                }
-               if (name == "flushleft" || name == "raggedright")
+               if (name == "flushleft")
                        parent_context.add_extra_stuff("\\align left\n");
-               else if (name == "flushright" || name == "raggedleft")
+               else if (name == "flushright")
                        parent_context.add_extra_stuff("\\align right\n");
-               else
+               else if (name == "center" || name == "centering")
                        parent_context.add_extra_stuff("\\align center\n");
+               else if (name == "singlespace")
+                       parent_context.add_extra_stuff("\\paragraph_spacing single\n");
+               else if (name == "onehalfspace")
+                       parent_context.add_extra_stuff("\\paragraph_spacing onehalf\n");
+               else if (name == "doublespace")
+                       parent_context.add_extra_stuff("\\paragraph_spacing double\n");
+               else if (name == "spacing")
+                       parent_context.add_extra_stuff("\\paragraph_spacing other " + p.verbatim_item() + "\n");
                parse_text(p, os, FLAG_END, outer, parent_context);
-               // Just in case the environment is empty ..
+               // Just in case the environment is empty
                parent_context.extra_stuff.erase();
                // We must begin a new paragraph to reset the alignment
                parent_context.new_paragraph(os);
@@ -807,7 +886,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
        }
 
        // The single '=' is meant here.
-       else if ((newlayout = findLayout(parent_context.textclass, name)).get() &&
+       else if ((newlayout = findLayout(parent_context.textclass, name)) &&
                  newlayout->isEnvironment()) {
                eat_whitespace(p, os, parent_context, false);
                Context context(true, parent_context.textclass, newlayout,
@@ -819,10 +898,32 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        context.need_end_deeper = true;
                }
                parent_context.check_end_layout(os);
+               if (last_env == name) {
+                       // we need to output a separator since LyX would export
+                       // the two environments as one otherwise (bug 5716)
+                       docstring const sep = from_ascii("--Separator--");
+                       TeX2LyXDocClass const & textclass(parent_context.textclass);
+                       if (textclass.hasLayout(sep)) {
+                               Context newcontext(parent_context);
+                               newcontext.layout = &(textclass[sep]);
+                               newcontext.check_layout(os);
+                               newcontext.check_end_layout(os);
+                       } else {
+                               parent_context.check_layout(os);
+                               begin_inset(os, "Note Note\n");
+                               os << "status closed\n";
+                               Context newcontext(true, textclass,
+                                               &(textclass.defaultLayout()));
+                               newcontext.check_layout(os);
+                               newcontext.check_end_layout(os);
+                               end_inset(os);
+                               parent_context.check_end_layout(os);
+                       }
+               }
                switch (context.layout->latextype) {
                case  LATEX_LIST_ENVIRONMENT:
-                       context.extra_stuff = "\\labelwidthstring "
-                               + p.verbatim_item() + '\n';
+                       context.add_par_extra_stuff("\\labelwidthstring "
+                                                   + p.verbatim_item() + '\n');
                        p.skip_spaces();
                        break;
                case  LATEX_BIB_ENVIRONMENT:
@@ -893,6 +994,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                parse_unknown_environment(p, name, os, FLAG_END, outer,
                                          parent_context);
 
+       last_env = name;
        active_environments.pop_back();
 }
 
@@ -900,7 +1002,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 /// parses a comment and outputs it to \p os.
 void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
 {
-       BOOST_ASSERT(t.cat() == catComment);
+       LASSERT(t.cat() == catComment, return);
        if (!t.cs().empty()) {
                context.check_layout(os);
                handle_comment(os, '%' + t.cs(), context);
@@ -1015,13 +1117,10 @@ string const normalize_filename(string const & name)
 /// convention (relative to .lyx file) if it is relative
 void fix_relative_filename(string & name)
 {
-       FileName fname(name);
-       if (fname.isAbsolute())
+       if (FileName::isAbsolute(name))
                return;
 
-       // FIXME UNICODE encoding of name may be wrong (makeAbsPath expects
-       // utf8)
-       name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFilename()),
+       name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFileName()),
                                   from_utf8(getParentFilePath())));
 }
 
@@ -1060,7 +1159,7 @@ void parse_noweb(Parser & p, ostream & os, Context & context)
        // always must be in an own paragraph.
        context.new_paragraph(os);
        Context newcontext(true, context.textclass,
-               context.textclass[from_ascii("Scrap")]);
+               &context.textclass[from_ascii("Scrap")]);
        newcontext.check_layout(os);
        os << name;
        while (p.good()) {
@@ -1094,19 +1193,126 @@ void parse_noweb(Parser & p, ostream & os, Context & context)
        newcontext.check_end_layout(os);
 }
 
+
+/// detects \\def, \\long\\def and \\global\\long\\def with ws and comments
+bool is_macro(Parser & p)
+{
+       Token first = p.curr_token();
+       if (first.cat() != catEscape || !p.good())
+               return false;
+       if (first.cs() == "def")
+               return true;
+       if (first.cs() != "global" && first.cs() != "long")
+               return false;
+       Token second = p.get_token();
+       int pos = 1;
+       while (p.good() && !p.isParagraph() && (second.cat() == catSpace ||
+              second.cat() == catNewline || second.cat() == catComment)) {
+               second = p.get_token();
+               pos++;
+       }
+       bool secondvalid = second.cat() == catEscape;
+       Token third;
+       bool thirdvalid = false;
+       if (p.good() && first.cs() == "global" && secondvalid &&
+           second.cs() == "long") {
+               third = p.get_token();
+               pos++;
+               while (p.good() && !p.isParagraph() &&
+                      (third.cat() == catSpace ||
+                       third.cat() == catNewline ||
+                       third.cat() == catComment)) {
+                       third = p.get_token();
+                       pos++;
+               }
+               thirdvalid = third.cat() == catEscape;
+       }
+       for (int i = 0; i < pos; ++i)
+               p.putback();
+       if (!secondvalid)
+               return false;
+       if (!thirdvalid)
+               return (first.cs() == "global" || first.cs() == "long") &&
+                      second.cs() == "def";
+       return first.cs() == "global" && second.cs() == "long" &&
+              third.cs() == "def";
+}
+
+
+/// Parse a macro definition (assumes that is_macro() returned true)
+void parse_macro(Parser & p, ostream & os, Context & context)
+{
+       context.check_layout(os);
+       Token first = p.curr_token();
+       Token second;
+       Token third;
+       string command = first.asInput();
+       if (first.cs() != "def") {
+               p.get_token();
+               eat_whitespace(p, os, context, false);
+               second = p.curr_token();
+               command += second.asInput();
+               if (second.cs() != "def") {
+                       p.get_token();
+                       eat_whitespace(p, os, context, false);
+                       third = p.curr_token();
+                       command += third.asInput();
+               }
+       }
+       eat_whitespace(p, os, context, false);
+       string const name = p.get_token().cs();
+       eat_whitespace(p, os, context, false);
+
+       // parameter text
+       bool simple = true;
+       string paramtext;
+       int arity = 0;
+       while (p.next_token().cat() != catBegin) {
+               if (p.next_token().cat() == catParameter) {
+                       // # found
+                       p.get_token();
+                       paramtext += "#";
+
+                       // followed by number?
+                       if (p.next_token().cat() == catOther) {
+                               char c = p.getChar();
+                               paramtext += c;
+                               // number = current arity + 1?
+                               if (c == arity + '0' + 1)
+                                       ++arity;
+                               else
+                                       simple = false;
+                       } else
+                               paramtext += p.get_token().cs();
+               } else {
+                       paramtext += p.get_token().cs();
+                       simple = false;
+               }
+       }
+
+       // only output simple (i.e. compatible) macro as FormulaMacros
+       string ert = '\\' + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
+       if (simple) {
+               context.check_layout(os);
+               begin_inset(os, "FormulaMacro");
+               os << "\n\\def" << ert;
+               end_inset(os);
+       } else
+               handle_ert(os, command + ert, context);
+}
+
 } // anonymous namespace
 
 
 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                Context & context)
 {
-       LayoutPtr newlayout;
-       // store the current selectlanguage to be used after \foreignlanguage
-       string selectlang;
+       Layout const * newlayout = 0;
        // Store the latest bibliographystyle (needed for bibtex inset)
        string bibliographystyle;
        bool const use_natbib = used_packages.find("natbib") != used_packages.end();
        bool const use_jurabib = used_packages.find("jurabib") != used_packages.end();
+       string last_env;
        while (p.good()) {
                Token const & t = p.get_token();
 
@@ -1132,6 +1338,14 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                if (t.character() == ']' && (flags & FLAG_BRACK_LAST))
                        return;
+               if (t.character() == '}' && (flags & FLAG_BRACE_LAST))
+                       return;
+
+               // If there is anything between \end{env} and \begin{env} we
+               // don't need to output a separator.
+               if (t.cat() != catSpace && t.cat() != catNewline &&
+                   t.asInput() != "\\begin")
+                       last_env = "";
 
                //
                // cat codes
@@ -1227,7 +1441,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                               t.cat() == catParameter) {
                        // This translates "&" to "\\&" which may be wrong...
                        context.check_layout(os);
-                       os << t.character();
+                       os << t.cs();
                }
 
                else if (p.isParagraph()) {
@@ -1243,10 +1457,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        if (t.character() == '~') {
                                if (context.layout->free_spacing)
                                        os << ' ';
-                               else
-                                       os << "\\InsetSpace ~\n";
+                               else {
+                                       begin_inset(os, "Space ~\n");
+                                       end_inset(os);
+                               }
                        } else
-                               os << t.character();
+                               os << t.cs();
                }
 
                else if (t.cat() == catBegin &&
@@ -1274,7 +1490,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                            next.character() == '*') {
                                p.get_token();
                                if (p.next_token().cat() == catEnd) {
-                                       os << next.character();
+                                       os << next.cs();
                                        p.get_token();
                                } else {
                                        p.putback();
@@ -1377,7 +1593,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                }
 
                else if (t.cs() == "begin")
-                       parse_environment(p, os, outer, context);
+                       parse_environment(p, os, outer, last_env, context);
 
                else if (t.cs() == "end") {
                        if (flags & FLAG_END) {
@@ -1414,7 +1630,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                        if (optarg) {
                                if (context.layout->labeltype != LABEL_MANUAL) {
-                                       // lyx does not support \item[\mybullet]
+                                       // LyX does not support \item[\mybullet]
                                        // in itemize environments
                                        handle_ert(os, "[", context);
                                        os << s;
@@ -1431,62 +1647,22 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "bibitem") {
                        context.set_item();
                        context.check_layout(os);
-                       os << "\\bibitem ";
-                       os << p.getOpt();
-                       os << '{' << p.verbatim_item() << '}' << "\n";
+                       begin_command_inset(os, "bibitem", "bibitem");
+                       os << "label \"" << p.getOptContent() << "\"\n";
+                       os << "key \"" << p.verbatim_item() << "\"\n";
+                       end_inset(os);
                }
 
-               else if (t.cs() == "def") {
-                       context.check_layout(os);
-                       eat_whitespace(p, os, context, false);
-                       string name = p.get_token().cs();
-                       eat_whitespace(p, os, context, false);
-
-                       // parameter text
-                       bool simple = true;
-                       string paramtext;
-                       int arity = 0;
-                       while (p.next_token().cat() != catBegin) {
-                               if (p.next_token().cat() == catParameter) {
-                                       // # found
-                                       p.get_token();
-                                       paramtext += "#";
-
-                                       // followed by number?
-                                       if (p.next_token().cat() == catOther) {
-                                               char c = p.getChar();
-                                               paramtext += c;
-                                               // number = current arity + 1?
-                                               if (c == arity + '0' + 1)
-                                                       ++arity;
-                                               else
-                                                       simple = false;
-                                       } else
-                                               paramtext += p.get_token().asString();
-                               } else {
-                                       paramtext += p.get_token().asString();
-                                       simple = false;
-                               }
-                       }
-
-                       // only output simple (i.e. compatible) macro as FormulaMacros
-                       string ert = "\\def\\" + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
-                       if (simple) {
-                               context.check_layout(os);
-                               begin_inset(os, "FormulaMacro");
-                               os << "\n" << ert;
-                               end_inset(os);
-                       } else
-                               handle_ert(os, ert, context);
-               }
+               else if (is_macro(p))
+                       parse_macro(p, os, context);
 
                else if (t.cs() == "noindent") {
                        p.skip_spaces();
-                       context.add_extra_stuff("\\noindent\n");
+                       context.add_par_extra_stuff("\\noindent\n");
                }
 
                else if (t.cs() == "appendix") {
-                       context.add_extra_stuff("\\start_of_appendix\n");
+                       context.add_par_extra_stuff("\\start_of_appendix\n");
                        // We need to start a new paragraph. Otherwise the
                        // appendix in 'bla\appendix\chapter{' would start
                        // too late.
@@ -1508,40 +1684,85 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        eat_whitespace(p, os, context, true);
                }
 
+               // Starred section headings
                // Must attempt to parse "Section*" before "Section".
                else if ((p.next_token().asInput() == "*") &&
                         context.new_layout_allowed &&
-                        // The single '=' is meant here.
-                        (newlayout = findLayout(context.textclass,
-                                                t.cs() + '*')).get() &&
+                        (newlayout = findLayout(context.textclass, t.cs() + '*')) &&
                         newlayout->isCommand()) {
+                       TeXFont const oldFont = context.font;
+                       // save the current font size
+                       string const size = oldFont.size;
+                       // reset the font size to default, because the
+                       // font size switches don't affect section
+                       // headings and the like
+                       context.font.size = known_coded_sizes[0];
+                       output_font_change(os, oldFont, context.font);
+                       // write the layout
                        p.get_token();
                        output_command_layout(os, p, outer, context, newlayout);
+                       // set the font size to the original value
+                       context.font.size = size;
+                       output_font_change(os, oldFont, context.font);
                        p.skip_spaces();
                }
 
-               // The single '=' is meant here.
+               // Section headings and the like
                else if (context.new_layout_allowed &&
-                        (newlayout = findLayout(context.textclass, t.cs())).get() &&
+                        (newlayout = findLayout(context.textclass, t.cs())) &&
                         newlayout->isCommand()) {
+                       TeXFont const oldFont = context.font;
+                       // save the current font size
+                       string const size = oldFont.size;
+                       // reset the font size to default, because the font size switches don't
+                       // affect section headings and the like
+                       context.font.size = known_coded_sizes[0];
+                       output_font_change(os, oldFont, context.font);
+                       // write the layout
                        output_command_layout(os, p, outer, context, newlayout);
+                       // set the font size to the original value
+                       context.font.size = size;
+                       output_font_change(os, oldFont, context.font);
                        p.skip_spaces();
                }
 
-               // Special handling for \caption
-               // FIXME: remove this when InsetCaption is supported.
-               else if (context.new_layout_allowed &&
-                        t.cs() == captionlayout->latexname()) {
-                       output_command_layout(os, p, outer, context, 
-                                             captionlayout);
+               else if (t.cs() == "caption") {
+                       // FIXME: this should get some cleanup. All
+                       // the \begin_layout:s are output by the
+                       // Context class!
                        p.skip_spaces();
+                       context.check_layout(os);
+                       p.skip_spaces();
+                       begin_inset(os, "Caption\n\n");
+                       os << "\\begin_layout " 
+                          << to_utf8(context.textclass.defaultLayout().name()) 
+                          << '\n';
+                       if (p.next_token().character() == '[') {
+                               p.get_token(); // eat '['
+                               begin_inset(os, "OptArg\n");
+                               os << "status collapsed\n";
+                               parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
+                               end_inset(os);
+                               eat_whitespace(p, os, context, false);
+                       }
+                       parse_text(p, os, FLAG_ITEM, outer, context);
+                       context.check_end_layout(os);
+                       // We don't need really a new paragraph, but
+                       // we must make sure that the next item gets a \begin_layout.
+                       context.new_paragraph(os);
+                       end_inset(os);
+                       p.skip_spaces();
+                       os << "\\end_layout\n";
                }
 
                else if (t.cs() == "includegraphics") {
                        bool const clip = p.next_token().asInput() == "*";
                        if (clip)
                                p.get_token();
-                       map<string, string> opts = split_map(p.getArg('[', ']'));
+                       string const arg = p.getArg('[', ']');
+                       map<string, string> opts;
+                       vector<string> keys;
+                       split_map(arg, opts, keys);
                        if (clip)
                                opts["clip"] = string();
                        string name = normalize_filename(p.verbatim_item());
@@ -1549,8 +1770,6 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        string const path = getMasterFilePath();
                        // We want to preserve relative / absolute filenames,
                        // therefore path is only used for testing
-                       // FIXME UNICODE encoding of name and path may be
-                       // wrong (makeAbsPath expects utf8)
                        if (!makeAbsPath(name, path).exists()) {
                                // The file extension is probably missing.
                                // Now try to find it out.
@@ -1581,8 +1800,6 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        name = pdftex_name;
                        }
 
-                       // FIXME UNICODE encoding of name and path may be
-                       // wrong (makeAbsPath expects utf8)
                        if (makeAbsPath(name, path).exists())
                                fix_relative_filename(name);
                        else
@@ -1605,9 +1822,20 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                val = val*100;
                                os << "\tscale " << val << '\n';
                        }
-                       if (opts.find("angle") != opts.end())
+                       if (opts.find("angle") != opts.end()) {
                                os << "\trotateAngle "
                                   << opts["angle"] << '\n';
+                               vector<string>::const_iterator a =
+                                       find(keys.begin(), keys.end(), "angle");
+                               vector<string>::const_iterator s =
+                                       find(keys.begin(), keys.end(), "width");
+                               if (s == keys.end())
+                                       s = find(keys.begin(), keys.end(), "height");
+                               if (s == keys.end())
+                                       s = find(keys.begin(), keys.end(), "scale");
+                               if (s != keys.end() && distance(s, a) > 0)
+                                       os << "\tscaleBeforeRotation\n";
+                       }
                        if (opts.find("origin") != opts.end()) {
                                ostringstream ss;
                                string const opt = opts["origin"];
@@ -1707,6 +1935,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.skip_spaces();
                        context.check_layout(os);
                        string const s = p.verbatim_item();
+                       //FIXME: this never triggers in UTF8
                        if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5")
                                os << s;
                        else
@@ -1714,47 +1943,37 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                           context);
                }
 
-               else if (t.cs() == "hfill") {
-                       context.check_layout(os);
-                       os << "\n\\hfill\n";
-                       skip_braces(p);
-                       p.skip_spaces();
-               }
-
                else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
                        // FIXME: Somehow prevent title layouts if
                        // "maketitle" was not found
-                       p.skip_spaces();
-                       skip_braces(p); // swallow this
+                       // swallow this
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "tableofcontents") {
-                       p.skip_spaces();
                        context.check_layout(os);
-                       begin_inset(os, "LatexCommand \\tableofcontents\n");
+                       begin_command_inset(os, "toc", "tableofcontents");
                        end_inset(os);
-                       skip_braces(p); // swallow this
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "listoffigures") {
-                       p.skip_spaces();
                        context.check_layout(os);
                        begin_inset(os, "FloatList figure\n");
                        end_inset(os);
-                       skip_braces(p); // swallow this
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "listoftables") {
-                       p.skip_spaces();
                        context.check_layout(os);
                        begin_inset(os, "FloatList table\n");
                        end_inset(os);
-                       skip_braces(p); // swallow this
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "listof") {
                        p.skip_spaces(true);
-                       string const name = p.get_token().asString();
+                       string const name = p.get_token().cs();
                        if (context.textclass.floats().typeExist(name)) {
                                context.check_layout(os);
                                begin_inset(os, "FloatList ");
@@ -1826,6 +2045,23 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                eat_whitespace(p, os, context, false);
                }
 
+               else if (t.cs() == "textcolor") {
+                       // scheme is \textcolor{color name}{text}
+                       string const color = p.verbatim_item();
+                       // we only support the predefined colors of the color package
+                       if (color == "black" || color == "blue" || color == "cyan"
+                               || color == "green" || color == "magenta" || color == "red"
+                               || color == "white" || color == "yellow") {
+                                       context.check_layout(os);
+                                       os << "\n\\color " << color << "\n";
+                                       parse_text_snippet(p, os, FLAG_ITEM, outer, context);
+                                       context.check_layout(os);
+                                       os << "\n\\color inherit\n";
+                       } else
+                               // for custom defined colors
+                               handle_ert(os, t.asInput() + "{" + color + "}", context);
+               }
+
                else if (t.cs() == "underbar") {
                        // Do NOT handle \underline.
                        // \underbar cuts through y, g, q, p etc.,
@@ -1845,6 +2081,21 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        os << "\n\\" << t.cs() << " default\n";
                }
 
+               else if (t.cs() == "lyxline") {
+                       context.check_layout(os);
+                       os << "\\lyxline";
+               }
+
+               else if (is_known(t.cs(), known_ref_commands)) {
+                       context.check_layout(os);
+                       begin_command_inset(os, "ref", t.cs());
+                       // LyX cannot handle newlines in a latex command
+                       // FIXME: Move the substitution into parser::getOpt()?
+                       os << subst(p.getOpt(), "\n", " ");
+                       os << "reference " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       end_inset(os);
+               }
+
                else if (use_natbib &&
                         is_known(t.cs(), known_natbib_commands) &&
                         ((t.cs() != "citefullauthor" &&
@@ -1852,19 +2103,14 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                           t.cs() != "citeyearpar") ||
                          p.next_token().asInput() != "*")) {
                        context.check_layout(os);
-                       // tex                       lyx
-                       // \citet[before][after]{a}  \citet[after][before]{a}
-                       // \citet[before][]{a}       \citet[][before]{a}
-                       // \citet[after]{a}          \citet[after]{a}
-                       // \citet{a}                 \citet{a}
-                       string command = '\\' + t.cs();
+                       string command = t.cs();
                        if (p.next_token().asInput() == "*") {
                                command += '*';
                                p.get_token();
                        }
-                       if (command == "\\citefullauthor")
+                       if (command == "citefullauthor")
                                // alternative name for "\\citeauthor*"
-                               command = "\\citeauthor*";
+                               command = "citeauthor*";
 
                        // text before the citation
                        string before;
@@ -1872,14 +2118,14 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        string after;
                        get_cite_arguments(p, true, before, after);
 
-                       if (command == "\\cite") {
+                       if (command == "cite") {
                                // \cite without optional argument means
                                // \citet, \cite with at least one optional
                                // argument means \citep.
                                if (before.empty() && after.empty())
-                                       command = "\\citet";
+                                       command = "citet";
                                else
-                                       command = "\\citep";
+                                       command = "citep";
                        }
                        if (before.empty() && after == "[]")
                                // avoid \citet[]{a}
@@ -1889,16 +2135,35 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                before.erase();
                                after.erase();
                        }
-                       begin_inset(os, "LatexCommand ");
-                       os << command << after << before
-                          << '{' << p.verbatim_item() << "}\n";
+                       // remove the brackets around after and before
+                       if (!after.empty()) {
+                               after.erase(0, 1);
+                               after.erase(after.length() - 1, 1);
+                               // LyX cannot handle newlines in the parameter
+                               after = subst(after, "\n", " ");
+                       }
+                       if (!before.empty()) {
+                               before.erase(0, 1);
+                               before.erase(before.length() - 1, 1);
+                               // LyX cannot handle newlines in the parameter
+                               before = subst(before, "\n", " ");
+                       }
+                       begin_command_inset(os, "citation", command);
+                       os << "after " << '"' << after << '"' << "\n";
+                       os << "before " << '"' << before << '"' << "\n";
+                       os << "key " << '"' << p.verbatim_item() << '"' << "\n";
                        end_inset(os);
                }
 
                else if (use_jurabib &&
-                        is_known(t.cs(), known_jurabib_commands)) {
+                        is_known(t.cs(), known_jurabib_commands) &&
+                        (t.cs() == "cite" || p.next_token().asInput() != "*")) {
                        context.check_layout(os);
-                       string const command = '\\' + t.cs();
+                       string command = t.cs();
+                       if (p.next_token().asInput() == "*") {
+                               command += '*';
+                               p.get_token();
+                       }
                        char argumentOrder = '\0';
                        vector<string> const & options = used_packages["jurabib"];
                        if (find(options.begin(), options.end(),
@@ -1924,24 +2189,87 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        "package options if you used an\n"
                                        "earlier jurabib version." << endl;
                        }
-                       begin_inset(os, "LatexCommand ");
-                       os << command << after << before
-                          << '{' << citation << "}\n";
+                       if (!after.empty()) {
+                               after.erase(0, 1);
+                               after.erase(after.length() - 1, 1);
+                       }
+                       if (!before.empty()) {
+                               before.erase(0, 1);
+                               before.erase(before.length() - 1, 1);
+                       }
+                       begin_command_inset(os, "citation", command);
+                       os << "after " << '"' << after << '"' << "\n";
+                       os << "before " << '"' << before << '"' << "\n";
+                       os << "key " << '"' << citation << '"' << "\n";
                        end_inset(os);
                }
 
-               else if (is_known(t.cs(), known_latex_commands)) {
-                       // This needs to be after the check for natbib and
-                       // jurabib commands, because "cite" has different
-                       // arguments with natbib and jurabib.
+               else if (t.cs() == "cite") {
                        context.check_layout(os);
-                       begin_inset(os, "LatexCommand ");
-                       os << '\\' << t.cs();
-                       // lyx cannot handle newlines in a latex command
-                       // FIXME: Move the substitution into parser::getOpt()?
-                       os << subst(p.getOpt(), "\n", " ");
-                       os << subst(p.getOpt(), "\n", " ");
-                       os << '{' << subst(p.verbatim_item(), "\n", " ") << "}\n";
+                       // LyX cannot handle newlines in a latex command
+                       string after = subst(p.getOptContent(), "\n", " ");
+                       begin_command_inset(os, "citation", "cite");
+                       os << "after " << '"' << after << '"' << "\n";
+                       os << "key " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       end_inset(os);
+               }
+
+               else if (t.cs() == "index") {
+                       context.check_layout(os);
+                       begin_inset(os, "Index\n");
+                       os << "status collapsed\n";
+                       parse_text_in_inset(p, os, FLAG_ITEM, false, context);
+                       end_inset(os);
+               }
+
+               else if (t.cs() == "nomenclature") {
+                       context.check_layout(os);
+                       begin_command_inset(os, "nomenclature", "nomenclature");
+                       // LyX cannot handle newlines in a latex command
+                       string prefix = subst(p.getOptContent(), "\n", " ");
+                       if (!prefix.empty())
+                               os << "prefix " << '"' << prefix << '"' << "\n";
+                       os << "symbol " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       os << "description " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       end_inset(os);
+               }
+               
+               else if (t.cs() == "label") {
+                       context.check_layout(os);
+                       begin_command_inset(os, "label", "label");
+                       // LyX cannot handle newlines in a latex command
+                       os << "name " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       end_inset(os);
+               }
+
+               else if (t.cs() == "printindex") {
+                       context.check_layout(os);
+                       begin_command_inset(os, "index_print", "printindex");
+                       end_inset(os);
+                       skip_spaces_braces(p);
+               }
+
+               else if (t.cs() == "printnomenclature") {
+                       context.check_layout(os);
+                       begin_command_inset(os, "nomencl_print", "printnomenclature");
+                       end_inset(os);
+                       skip_spaces_braces(p);
+               }
+
+               else if (t.cs() == "url") {
+                       context.check_layout(os);
+                       begin_inset(os, "Flex URL\n");
+                       os << "status collapsed\n";
+                       parse_text_in_inset(p, os, FLAG_ITEM, false, context);
+                       end_inset(os);
+               }
+
+               else if (LYX_FORMAT >= 408 &&
+                        (t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
+                       context.check_layout(os);
+                       begin_inset(os, "script ");
+                       os << t.cs().substr(4) << '\n';
+                       parse_text_in_inset(p, os, FLAG_ITEM, false, context);
                        end_inset(os);
                }
 
@@ -1963,7 +2291,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        char const * const * where = is_known(t.cs(), known_sizes);
                        context.check_layout(os);
                        TeXFont const oldFont = context.font;
-                       context.font.size = known_coded_sizes[where - known_sizes];
+                       // the font size index differs by 1, because the known_coded_sizes
+                       // has additionally a "default" entry
+                       context.font.size = known_coded_sizes[where - known_sizes + 1];
                        output_font_change(os, oldFont, context.font);
                        eat_whitespace(p, os, context, false);
                }
@@ -2047,53 +2377,75 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                else if (t.cs() == "selectlanguage") {
                        context.check_layout(os);
-                       // save the language for the case that a \foreignlanguage is used 
-                       selectlang = subst(p.verbatim_item(), "\n", " ");
-                       os << "\\lang " << selectlang << "\n";
-                       
+                       // save the language for the case that a
+                       // \foreignlanguage is used 
+
+                       context.font.language = subst(p.verbatim_item(), "\n", " ");
+                       os << "\\lang " << context.font.language << "\n";
                }
 
                else if (t.cs() == "foreignlanguage") {
                        context.check_layout(os);
                        os << "\n\\lang " << subst(p.verbatim_item(), "\n", " ") << "\n";
                        os << subst(p.verbatim_item(), "\n", " ");
+                       // FIXME: the second argument of selectlanguage
+                       // has to be parsed (like for \textsf, for
+                       // example). 
                        // set back to last selectlanguage
-                       os << "\n\\lang " << selectlang << "\n";
+                       os << "\n\\lang " << context.font.language << "\n";
+               }
+
+               else if (t.cs() == "inputencoding") {
+                       // nothing to write here
+                       string const enc = subst(p.verbatim_item(), "\n", " ");
+                       p.setEncoding(enc);
                }
 
-               else if (t.cs() == "inputencoding")
-                       // write nothing because this is done by LyX using the "\lang"
-                       // information given by selectlanguage and foreignlanguage
-                       subst(p.verbatim_item(), "\n", " ");
-               
                else if (t.cs() == "LyX" || t.cs() == "TeX"
                         || t.cs() == "LaTeX") {
                        context.check_layout(os);
                        os << t.cs();
-                       skip_braces(p); // eat {}
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "LaTeXe") {
                        context.check_layout(os);
                        os << "LaTeX2e";
-                       skip_braces(p); // eat {}
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "ldots") {
                        context.check_layout(os);
-                       skip_braces(p);
                        os << "\\SpecialChar \\ldots{}\n";
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "lyxarrow") {
                        context.check_layout(os);
                        os << "\\SpecialChar \\menuseparator\n";
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "textcompwordmark") {
                        context.check_layout(os);
                        os << "\\SpecialChar \\textcompwordmark{}\n";
+                       skip_spaces_braces(p);
+               }
+
+               else if (t.cs() == "slash") {
+                       context.check_layout(os);
+                       os << "\\SpecialChar \\slash{}\n";
+                       skip_spaces_braces(p);
+               }
+
+               else if (t.cs() == "nobreakdash") {
+                       context.check_layout(os);
+                       os << "\\SpecialChar \\nobreakdash\n";
+               }
+
+               else if (t.cs() == "textquotedbl") {
+                       context.check_layout(os);
+                       os << "\"";
                        skip_braces(p);
                }
 
@@ -2111,19 +2463,19 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "textasciitilde") {
                        context.check_layout(os);
                        os << '~';
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "textasciicircum") {
                        context.check_layout(os);
                        os << '^';
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "textbackslash") {
                        context.check_layout(os);
                        os << "\n\\backslash\n";
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#"
@@ -2158,52 +2510,31 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        handle_ert(os, oss.str(), context);
                }
 
-               else if (t.cs() == "\"") {
-                       context.check_layout(os);
-                       string const name = p.verbatim_item();
-                            if (name == "a") os << '\xe4';
-                       else if (name == "o") os << '\xf6';
-                       else if (name == "u") os << '\xfc';
-                       else if (name == "A") os << '\xc4';
-                       else if (name == "O") os << '\xd6';
-                       else if (name == "U") os << '\xdc';
-                       else handle_ert(os, "\"{" + name + "}", context);
-               }
-
                // Problem: \= creates a tabstop inside the tabbing environment
                // and else an accent. In the latter case we really would want
                // \={o} instead of \= o.
                else if (t.cs() == "=" && (flags & FLAG_TABBING))
                        handle_ert(os, t.asInput(), context);
 
-               else if (t.cs() == "H" || t.cs() == "c" || t.cs() == "^"
-                        || t.cs() == "'" || t.cs() == "`"
-                        || t.cs() == "~" || t.cs() == "." || t.cs() == "=") {
-                       // we need the trim as the LyX parser chokes on such spaces
-                       // The argument of InsetLatexAccent is parsed as a
-                       // subset of LaTeX, so don't parse anything here,
-                       // but use the raw argument.
-                       // Otherwise we would convert \~{\i} wrongly.
-                       // This will of course not translate \~{\ss} to \~{ß},
-                       // but that does at least compile and does only look
-                       // strange on screen.
-                       context.check_layout(os);
-                       os << "\\i \\" << t.cs() << "{"
-                          << trim(p.verbatim_item(), " ")
-                          << "}\n";
-               }
-
-               else if (t.cs() == "ss") {
-                       context.check_layout(os);
-                       os << "\xdf";
-                       skip_braces(p); // eat {}
-               }
-
-               else if (t.cs() == "i" || t.cs() == "j" || t.cs() == "l" ||
-                        t.cs() == "L") {
-                       context.check_layout(os);
-                       os << "\\i \\" << t.cs() << "{}\n";
-                       skip_braces(p); // eat {}
+               // accents (see Table 6 in Comprehensive LaTeX Symbol List)
+               else if (t.cs().size() == 1 
+                        && contains("\"'.=^`bcdHkrtuv~", t.cs())) {
+                       context.check_layout(os);
+                       // try to see whether the string is in unicodesymbols
+                       docstring rem;
+                       string command = t.asInput() + "{" 
+                               + trim(p.verbatim_item())
+                               + "}";
+                       docstring s = encodings.fromLaTeXCommand(from_utf8(command), rem);
+                       if (!s.empty()) {
+                               if (!rem.empty())
+                                       cerr << "When parsing " << command 
+                                            << ", result is " << to_utf8(s)
+                                            << "+" << to_utf8(rem) << endl;
+                               os << to_utf8(s);
+                       } else
+                               // we did not find a non-ert version
+                               handle_ert(os, command, context);
                }
 
                else if (t.cs() == "\\") {
@@ -2220,53 +2551,24 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                }
 
-               else if (t.cs() == "newline" ||
-                       t.cs() == "linebreak") {
+               else if (t.cs() == "newline"
+                       || t.cs() == "linebreak") {
                        context.check_layout(os);
                        os << "\n\\" << t.cs() << "\n";
-                       skip_braces(p); // eat {}
-               }
-
-               else if (t.cs() == "href") {
-                       context.check_layout(os);
-                       begin_inset(os, "CommandInset ");
-                       os << t.cs() << "\n";
-                       os << "LatexCommand " << t.cs() << "\n";
-                       bool erase = false;
-                       size_t pos;
-                       // the first argument is "type:target", "type:" is optional
-                       // the second argument the name
-                       string href_target = subst(p.verbatim_item(), "\n", " ");
-                       string href_name = subst(p.verbatim_item(), "\n", " ");
-                       string href_type;
-                       // serach for the ":" to divide type from target
-                       if ((pos = href_target.find(":", 0)) != string::npos){
-                               href_type = href_target;
-                               href_type.erase(pos + 1, href_type.length());
-                               href_target.erase(0, pos + 1);
-                           erase = true;                                                                                       
-                       }
-                       os << "name " << '"' << href_name << '"' << "\n";
-                       os << "target " << '"' << href_target << '"' << "\n";
-                       if(erase)
-                               os << "type " << '"' << href_type << '"' << "\n";
-                       end_inset(os);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "input" || t.cs() == "include"
                         || t.cs() == "verbatiminput") {
-                       string name = '\\' + t.cs();
+                       string name = t.cs();
                        if (t.cs() == "verbatiminput"
                            && p.next_token().asInput() == "*")
                                name += p.get_token().asInput();
                        context.check_layout(os);
-                       begin_inset(os, "Include ");
                        string filename(normalize_filename(p.getArg('{', '}')));
                        string const path = getMasterFilePath();
                        // We want to preserve relative / absolute filenames,
                        // therefore path is only used for testing
-                       // FIXME UNICODE encoding of filename and path may be
-                       // wrong (makeAbsPath expects utf8)
                        if ((t.cs() == "include" || t.cs() == "input") &&
                            !makeAbsPath(filename, path).exists()) {
                                // The file extension is probably missing.
@@ -2277,28 +2579,53 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                if (!tex_name.empty())
                                        filename = tex_name;
                        }
-                       // FIXME UNICODE encoding of filename and path may be
-                       // wrong (makeAbsPath expects utf8)
+                       bool xfig = false;
+                       string outname;
                        if (makeAbsPath(filename, path).exists()) {
                                string const abstexname =
-                                       makeAbsPath(filename, path).absFilename();
+                                       makeAbsPath(filename, path).absFileName();
                                string const abslyxname =
                                        changeExtension(abstexname, ".lyx");
+                               string const absfigname =
+                                       changeExtension(abstexname, ".fig");
                                fix_relative_filename(filename);
                                string const lyxname =
                                        changeExtension(filename, ".lyx");
-                               if (t.cs() != "verbatiminput" &&
-                                   tex2lyx(abstexname, FileName(abslyxname))) {
-                                       os << name << '{' << lyxname << "}\n";
+                               if (t.cs() == "input" && FileName(absfigname).exists()) {
+                                       FileName const absepsname(
+                                               changeExtension(abstexname, ".eps"));
+                                       FileName const abspdfname(
+                                               changeExtension(abstexname, ".pdf"));
+                                       string const ext = getExtension(abstexname);
+                                       bool const xfigpdf =
+                                               abspdfname.exists() && ext == "pdftex_t";
+                                       bool const xfigeps  =
+                                               absepsname.exists() && ext == "pstex_t";
+                                       xfig = xfigpdf || xfigeps;
+                               }
+                               if (xfig) {
+                                       outname = changeExtension(filename, ".fig");
+                               } else if (t.cs() != "verbatiminput" &&
+                                   tex2lyx(abstexname, FileName(abslyxname),
+                                           p.getEncoding())) {
+                                       outname = lyxname;
                                } else {
-                                       os << name << '{' << filename << "}\n";
+                                       outname = filename;
                                }
                        } else {
                                cerr << "Warning: Could not find included file '"
                                     << filename << "'." << endl;
-                               os << name << '{' << filename << "}\n";
+                               outname = filename;
+                       }
+                       if (xfig) {
+                               begin_inset(os, "External\n");
+                               os << "\ttemplate XFig\n"
+                                  << "\tfilename " << outname << '\n';
+                       } else {
+                               begin_command_inset(os, "include", name);
+                               os << "preview false\n"
+                                     "filename \"" << outname << "\"\n";
                        }
-                       os << "preview false\n";
                        end_inset(os);
                }
 
@@ -2312,18 +2639,29 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                else if (t.cs() == "bibliography") {
                        context.check_layout(os);
-                       begin_inset(os, "LatexCommand ");
-                       os << "\\bibtex";
+                       begin_command_inset(os, "bibtex", "bibtex");
+                       os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n";
                        // Do we have a bibliographystyle set?
-                       if (!bibliographystyle.empty()) {
-                               os << '[' << bibliographystyle << ']';
-                       }
-                       os << '{' << p.verbatim_item() << "}\n";
+                       if (!bibliographystyle.empty())
+                               os << "options " << '"' << bibliographystyle << '"' << "\n";
                        end_inset(os);
                }
 
                else if (t.cs() == "parbox")
                        parse_box(p, os, FLAG_ITEM, outer, context, true);
+               
+               //\makebox() is part of the picture environment and different from \makebox{}
+               //\makebox{} will be parsed by parse_box when bug 2956 is fixed
+               else if (t.cs() == "makebox") {
+                       string arg = t.asInput();
+                       if (p.next_token().character() == '(')
+                               //the syntax is: \makebox(x,y)[position]{content}
+                               arg += p.getFullParentheseArg();
+                       else
+                               //the syntax is: \makebox[width][position]{content}
+                               arg += p.getFullOpt();
+                       handle_ert(os, arg + p.getFullOpt(), context);
+               }
 
                else if (t.cs() == "smallskip" ||
                         t.cs() == "medskip" ||
@@ -2333,15 +2671,16 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        begin_inset(os, "VSpace ");
                        os << t.cs();
                        end_inset(os);
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (is_known(t.cs(), known_spaces)) {
                        char const * const * where = is_known(t.cs(), known_spaces);
                        context.check_layout(os);
-                       begin_inset(os, "InsetSpace ");
+                       begin_inset(os, "Space ");
                        os << '\\' << known_coded_spaces[where - known_spaces]
                           << '\n';
+                       end_inset(os);
                        // LaTeX swallows whitespace after all spaces except
                        // "\\,". We have to do that here, too, because LyX
                        // adds "{}" which would make the spaces significant.
@@ -2361,15 +2700,14 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        t.cs() == "cleardoublepage") {
                        context.check_layout(os);
                        os << "\n\\" << t.cs() << "\n";
-                       skip_braces(p); // eat {}
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "newcommand" ||
                         t.cs() == "providecommand" ||
-                        t.cs() == "renewcommand" ||
-                        t.cs() == "newlyxcommand") {
-                       // these could be handled by parse_command(), but
-                       // we need to call add_known_command() here.
+                        t.cs() == "renewcommand") {
+                       // providecommand could be handled by parse_command(),
+                       // but we need to call add_known_command() here.
                        string name = t.asInput();
                        if (p.next_token().asInput() == "*") {
                                // Starred form. Eat '*'
@@ -2378,35 +2716,34 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                        string const command = p.verbatim_item();
                        string const opt1 = p.getOpt();
-                       string optionals;
-                       unsigned optionalsNum = 0;
-                       while (true) {
-                               string const opt = p.getFullOpt();
-                               if (opt.empty())
-                                       break;
-                               optionalsNum++;
-                               optionals += opt;
-                       }
-                       add_known_command(command, opt1, optionalsNum);
-                       string const ert = name + '{' + command + '}' + opt1
-                               + optionals + '{' + p.verbatim_item() + '}';
+                       string const opt2 = p.getFullOpt();
+                       add_known_command(command, opt1, !opt2.empty());
+                       string const ert = name + '{' + command + '}' +
+                                          opt1 + opt2 +
+                                          '{' + p.verbatim_item() + '}';
 
-                       context.check_layout(os);
-                       begin_inset(os, "FormulaMacro");
-                       os << "\n" << ert;
-                       end_inset(os);
+                       if (t.cs() == "providecommand")
+                               handle_ert(os, ert, context);
+                       else {
+                               context.check_layout(os);
+                               begin_inset(os, "FormulaMacro");
+                               os << "\n" << ert;
+                               end_inset(os);
+                       }
                }
-
-               else if (t.cs() == "vspace") {
+               
+               else if (t.cs() == "hspace" || t.cs() == "vspace") {
                        bool starred = false;
                        if (p.next_token().asInput() == "*") {
                                p.get_token();
                                starred = true;
                        }
+                       string name = t.asInput();
                        string const length = p.verbatim_item();
                        string unit;
                        string valstring;
                        bool valid = splitLatexLength(length, valstring, unit);
+                       bool known_hspace = false;
                        bool known_vspace = false;
                        bool known_unit = false;
                        double value;
@@ -2414,21 +2751,31 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                istringstream iss(valstring);
                                iss >> value;
                                if (value == 1.0) {
-                                       if (unit == "\\smallskipamount") {
-                                               unit = "smallskip";
-                                               known_vspace = true;
-                                       } else if (unit == "\\medskipamount") {
-                                               unit = "medskip";
-                                               known_vspace = true;
-                                       } else if (unit == "\\bigskipamount") {
-                                               unit = "bigskip";
-                                               known_vspace = true;
-                                       } else if (unit == "\\fill") {
-                                               unit = "vfill";
-                                               known_vspace = true;
+                                       if (t.cs()[0] == 'h') {
+                                               if (unit == "\\fill") {
+                                                       if (!starred) {
+                                                               unit = "";
+                                                               name = "hfill";
+                                                       }
+                                                       known_hspace = true;
+                                               }
+                                       } else {
+                                               if (unit == "\\smallskipamount") {
+                                                       unit = "smallskip";
+                                                       known_vspace = true;
+                                               } else if (unit == "\\medskipamount") {
+                                                       unit = "medskip";
+                                                       known_vspace = true;
+                                               } else if (unit == "\\bigskipamount") {
+                                                       unit = "bigskip";
+                                                       known_vspace = true;
+                                               } else if (unit == "\\fill") {
+                                                       unit = "vfill";
+                                                       known_vspace = true;
+                                               }
                                        }
                                }
-                               if (!known_vspace) {
+                               if (!known_hspace && !known_vspace) {
                                        switch (unitFromString(unit)) {
                                        case Length::SP:
                                        case Length::PT:
@@ -2450,8 +2797,23 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                }
                        }
 
-                       if (known_unit || known_vspace) {
-                               // Literal length or known variable
+                       if (t.cs()[0] == 'h' && (known_unit || known_hspace)) {
+                               // Literal horizontal length or known variable
+                               context.check_layout(os);
+                               begin_inset(os, "Space \\");
+                               os << name;
+                               if (starred)
+                                       os << '*';
+                               os << '{';
+                               if (known_hspace)
+                                       os << unit;
+                               os << "}\n";
+                               if (known_unit && !known_hspace)
+                                       os << "\\length "
+                                          << translate_len(length) << '\n';
+                               end_inset(os);
+                       } else if (known_unit || known_vspace) {
+                               // Literal vertical length or known variable
                                context.check_layout(os);
                                begin_inset(os, "VSpace ");
                                if (known_unit)
@@ -2461,8 +2823,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        os << '*';
                                end_inset(os);
                        } else {
-                               // LyX can't handle other length variables in Inset VSpace
-                               string name = t.asInput();
+                               // LyX can't handle other length variables in Inset V?Space
                                if (starred)
                                        name += '*';
                                if (valid) {
@@ -2478,6 +2839,21 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                }
 
                else {
+                       // try to see whether the string is in unicodesymbols
+                       // Only use text mode commands, since we are in text mode here,
+                       // and math commands may be invalid (bug 6797)
+                       docstring rem;
+                       docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
+                                                                rem, Encodings::TEXT_CMD);
+                       if (!s.empty()) {
+                               if (!rem.empty())
+                                       cerr << "When parsing " << t.cs() 
+                                            << ", result is " << to_utf8(s)
+                                            << "+" << to_utf8(rem) << endl;
+                               context.check_layout(os);
+                               os << to_utf8(s);
+                               skip_spaces_braces(p);
+                       }
                        //cerr << "#: " << t << " mode: " << mode << endl;
                        // heuristic: read up to next non-nested space
                        /*
@@ -2491,14 +2867,16 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        cerr << "found ERT: " << s << endl;
                        handle_ert(os, s + ' ', context);
                        */
-                       string name = t.asInput();
-                       if (p.next_token().asInput() == "*") {
-                               // Starred commands like \vspace*{}
-                               p.get_token();                          // Eat '*'
-                               name += '*';
+                       else {
+                               string name = t.asInput();
+                               if (p.next_token().asInput() == "*") {
+                                       // Starred commands like \vspace*{}
+                                       p.get_token();  // Eat '*'
+                                       name += '*';
+                               }
+                               if (!parse_command(name, p, os, outer, context))
+                                       handle_ert(os, name, context);
                        }
-                       if (! parse_command(name, p, os, outer, context))
-                               handle_ert(os, name, context);
                }
 
                if (flags & FLAG_LEAVE) {