]> git.lyx.org Git - lyx.git/blobdiff - src/tex2lyx/text.cpp
tex2lyx/text.cpp: code simplification
[lyx.git] / src / tex2lyx / text.cpp
index 218b6cbcee88a3dbefa04f1b418c47cce1ce2693..8a9951d3c713acad8f59ff05cb7c5987de10deee 100644 (file)
@@ -41,10 +41,15 @@ namespace lyx {
 
 
 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
-               Context const & context)
+               Context const & context, InsetLayout const * layout)
 {
+       bool const forcePlainLayout =
+               layout ? layout->forcePlainLayout() : false;
        Context newcontext(true, context.textclass);
-       newcontext.font = context.font;
+       if (forcePlainLayout)
+               newcontext.layout = &context.textclass.plainLayout();
+       else
+               newcontext.font = context.font;
        parse_text(p, os, flags, outer, newcontext);
        newcontext.check_end_layout(os);
 }
@@ -52,6 +57,17 @@ void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
 
 namespace {
 
+void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
+               Context const & context, string const & name)
+{
+       InsetLayout const * layout = 0;
+       DocumentClass::InsetLayouts::const_iterator it =
+               context.textclass.insetLayouts().find(from_ascii(name));
+       if (it != context.textclass.insetLayouts().end())
+               layout = &(it->second);
+       parse_text_in_inset(p, os, flags, outer, context, layout);
+}
+
 /// parses a paragraph snippet, useful for example for \\emph{...}
 void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
                Context & context)
@@ -93,9 +109,13 @@ string parse_text_snippet(Parser & p, unsigned flags, const bool outer,
 char const * const known_ref_commands[] = { "ref", "pageref", "vref",
  "vpageref", "prettyref", "eqref", 0 };
 
+char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref",
+ "vpageref", "formatted", "eqref", 0 };
+
 /*!
  * natbib commands.
- * The starred forms are also known.
+ * The starred forms are also known except for "citefullauthor",
+ * "citeyear" and "citeyearpar".
  */
 char const * const known_natbib_commands[] = { "cite", "citet", "citep",
 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
@@ -111,7 +131,7 @@ char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
 // "fullcite",
 // "footcite", "footcitet", "footcitep", "footcitealt", "footcitealp",
 // "footciteauthor", "footciteyear", "footciteyearpar",
-"citefield", "citetitle", "cite*", 0 };
+"citefield", "citetitle", 0 };
 
 /// LaTeX names for quotes
 char const * const known_quotes[] = { "dq", "guillemotleft", "flqq", "og",
@@ -129,8 +149,8 @@ char const * const known_coded_quotes[] = { "prd", "ard", "ard", "ard",
 char const * const known_sizes[] = { "tiny", "scriptsize", "footnotesize",
 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
 
-/// the same as known_sizes with .lyx names plus a default entry
-char const * const known_coded_sizes[] = { "default", "tiny", "scriptsize", "footnotesize",
+/// the same as known_sizes with .lyx names
+char const * const known_coded_sizes[] = { "tiny", "scriptsize", "footnotesize",
 "small", "normal", "large", "larger", "largest", "huge", "giant", 0};
 
 /// LaTeX 2.09 names for font families
@@ -190,28 +210,40 @@ char const * const known_pdftex_graphics_formats[] = {"png", "pdf", "jpg",
 char const * const known_tex_extensions[] = {"tex", 0};
 
 /// spaces known by InsetSpace
-char const * const known_spaces[] = { " ", "space", ",", "thinspace", "quad",
-"qquad", "enspace", "enskip", "negthinspace", 0};
+char const * const known_spaces[] = { " ", "space", ",",
+"thinspace", "quad", "qquad", "enspace", "enskip",
+"negthinspace", "negmedspace", "negthickspace", "textvisiblespace",
+"hfill", "dotfill", "hrulefill", "leftarrowfill", "rightarrowfill",
+"upbracefill", "downbracefill", 0};
 
 /// the same as known_spaces with .lyx names
 char const * const known_coded_spaces[] = { "space{}", "space{}",
 "thinspace{}", "thinspace{}", "quad{}", "qquad{}", "enspace{}", "enskip{}",
-"negthinspace{}", 0};
+"negthinspace{}", "negmedspace{}", "negthickspace{}", "textvisiblespace{}",
+"hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}", "rightarrowfill{}",
+"upbracefill{}", "downbracefill{}", 0};
 
+/// These are translated by LyX to commands like "\\LyX{}", so we have to put
+/// them in ERT. "LaTeXe" must come before "LaTeX"!
+char const * const known_phrases[] = {"LyX", "TeX", "LaTeXe", "LaTeX", 0};
+char const * const known_coded_phrases[] = {"LyX", "TeX", "LaTeX2e", "LaTeX", 0};
+int const known_phrase_lengths[] = {3, 5, 7, 0};
 
-/// splits "x=z, y=b" into a map
-map<string, string> split_map(string const & s)
+
+/// splits "x=z, y=b" into a map and an ordered keyword vector
+void split_map(string const & s, map<string, string> & res, vector<string> & keys)
 {
-       map<string, string> res;
        vector<string> v;
        split(s, v);
+       res.clear();
+       keys.resize(v.size());
        for (size_t i = 0; i < v.size(); ++i) {
                size_t const pos   = v[i].find('=');
-               string const index = v[i].substr(0, pos);
-               string const value = v[i].substr(pos + 1, string::npos);
-               res[trim(index)] = trim(value);
+               string const index = trimSpaceAndEol(v[i].substr(0, pos));
+               string const value = trimSpaceAndEol(v[i].substr(pos + 1, string::npos));
+               res[index] = value;
+               keys[i] = index;
        }
-       return res;
 }
 
 
@@ -239,20 +271,20 @@ bool splitLatexLength(string const & len, string & value, string & unit)
                        return false;
                }
        } else {
-               value = trim(string(length, 0, i));
+               value = trimSpaceAndEol(string(length, 0, i));
        }
        if (value == "-")
                value = "-1.0";
        // 'cM' is a valid LaTeX length unit. Change it to 'cm'
        if (contains(len, '\\'))
-               unit = trim(string(len, i));
+               unit = trimSpaceAndEol(string(len, i));
        else
-               unit = ascii_lowercase(trim(string(len, i)));
+               unit = ascii_lowercase(trimSpaceAndEol(string(len, i)));
        return true;
 }
 
 
-/// A simple function to translate a latex length to something lyx can
+/// A simple function to translate a latex length to something LyX can
 /// understand. Not perfect, but rather best-effort.
 bool translate_len(string const & length, string & valstring, string & unit)
 {
@@ -354,11 +386,12 @@ void begin_inset(ostream & os, string const & name)
        os << "\n\\begin_inset " << name;
 }
 
+
 void begin_command_inset(ostream & os, string const & name,
-                                                string const & latexname)
+                         string const & latexname)
 {
-       os << "\n\\begin_inset CommandInset " << name;
-       os << "\nLatexCommand " << latexname << "\n";
+       begin_inset(os, "CommandInset ");
+       os << name << "\nLatexCommand " << latexname << '\n';
 }
 
 
@@ -368,16 +401,64 @@ void end_inset(ostream & os)
 }
 
 
-void skip_braces(Parser & p)
+bool skip_braces(Parser & p)
 {
        if (p.next_token().cat() != catBegin)
-               return;
+               return false;
        p.get_token();
        if (p.next_token().cat() == catEnd) {
                p.get_token();
-               return;
+               return true;
        }
        p.putback();
+       return false;
+}
+
+
+/// replace LaTeX commands in \p s from the unicodesymbols file with their
+/// unicode points
+docstring convert_unicodesymbols(docstring s)
+{
+       odocstringstream os;
+       for (size_t i = 0; i < s.size();) {
+               if (s[i] != '\\') {
+                       os.put(s[i++]);
+                       continue;
+               }
+               s = s.substr(i);
+               docstring rem;
+               docstring parsed = encodings.fromLaTeXCommand(s, rem,
+                               Encodings::TEXT_CMD);
+               os << parsed;
+               s = rem;
+               if (s.empty() || s[0] != '\\')
+                       i = 0;
+               else
+                       i = 1;
+       }
+       return os.str();
+}
+
+
+/// try to convert \p s to a valid InsetCommand argument
+string convert_command_inset_arg(string s)
+{
+       if (isAscii(s))
+               // since we don't know the input encoding we can't use from_utf8
+               s = to_utf8(convert_unicodesymbols(from_ascii(s)));
+       // LyX cannot handle newlines in a latex command
+       return subst(s, "\n", " ");
+}
+
+
+void handle_backslash(ostream & os, string const & s)
+{
+       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
+               if (*it == '\\')
+                       os << "\n\\backslash\n";
+               else
+                       os << *it;
+       }
 }
 
 
@@ -410,12 +491,7 @@ void handle_comment(ostream & os, string const & s, Context & context)
        begin_inset(os, "ERT");
        os << "\nstatus collapsed\n";
        newcontext.check_layout(os);
-       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
-               if (*it == '\\')
-                       os << "\n\\backslash\n";
-               else
-                       os << *it;
-       }
+       handle_backslash(os, s);
        // make sure that our comment is the last thing on the line
        newcontext.new_paragraph(os);
        newcontext.check_layout(os);
@@ -424,31 +500,55 @@ void handle_comment(ostream & os, string const & s, Context & context)
 }
 
 
-Layout const * findLayout(TextClass const & textclass, string const & name)
+Layout const * findLayout(TextClass const & textclass, string const & name, bool command)
+{
+       Layout const * layout = findLayoutWithoutModule(textclass, name, command);
+       if (layout)
+               return layout;
+       if (checkModule(name, command))
+               return findLayoutWithoutModule(textclass, name, command);
+       return layout;
+}
+
+
+InsetLayout const * findInsetLayout(TextClass const & textclass, string const & name, bool command)
 {
-       DocumentClass::const_iterator lit = textclass.begin();
-       DocumentClass::const_iterator len = textclass.end();
-       for (; lit != len; ++lit)
-               if (lit->latexname() == name)
-                       return &*lit;
-       return 0;
+       InsetLayout const * insetlayout = findInsetLayoutWithoutModule(textclass, name, command);
+       if (insetlayout)
+               return insetlayout;
+       if (checkModule(name, command))
+               return findInsetLayoutWithoutModule(textclass, name, command);
+       return insetlayout;
 }
 
 
 void eat_whitespace(Parser &, ostream &, Context &, bool);
 
 
-Layout * captionlayout()
+/*!
+ * Skips whitespace and braces.
+ * This should be called after a command has been parsed that is not put into
+ * ERT, and where LyX adds "{}" if needed.
+ */
+void skip_spaces_braces(Parser & p, bool keepws = false)
 {
-       static Layout * lay = 0;
-       if (!lay) {
-               lay = new Layout;
-               lay->setName(from_ascii("Caption"));
-               lay->setLatexName("caption");
-               lay->latextype = LATEX_COMMAND;
-               lay->optionalargs = 1;
-       }
-       return lay;
+       /* The following four examples produce the same typeset output and
+          should be handled by this function:
+          - abc \j{} xyz
+          - abc \j {} xyz
+          - abc \j 
+            {} xyz
+          - abc \j %comment
+            {} xyz
+        */
+       // Unfortunately we need to skip comments, too.
+       // We can't use eat_whitespace since writing them after the {}
+       // results in different output in some cases.
+       bool const skipped_spaces = p.skip_spaces(true);
+       bool const skipped_braces = skip_braces(p);
+       if (keepws && skipped_spaces && !skipped_braces)
+               // put back the space (it is better handled by check_space)
+               p.unskip_spaces(true);
 }
 
 
@@ -456,6 +556,15 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
                           Context & parent_context,
                           Layout const * newlayout)
 {
+       TeXFont const oldFont = parent_context.font;
+       // save the current font size
+       string const size = oldFont.size;
+       // reset the font size to default, because the font size switches
+       // don't affect section headings and the like
+       parent_context.font.size = Context::normalfont.size;
+       // we only need to write the font change if we have an open layout
+       if (!parent_context.atParagraphStart())
+               output_font_change(os, oldFont, parent_context.font);
        parent_context.check_end_layout(os);
        Context context(true, parent_context.textclass, newlayout,
                        parent_context.layout, parent_context.font);
@@ -467,16 +576,32 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
        }
        context.check_deeper(os);
        context.check_layout(os);
-       if (context.layout->optionalargs > 0) {
+       unsigned int optargs = 0;
+       while (optargs < context.layout->optargs) {
                eat_whitespace(p, os, context, false);
-               if (p.next_token().character() == '[') {
-                       p.get_token(); // eat '['
-                       begin_inset(os, "OptArg\n");
-                       os << "status collapsed\n\n";
-                       parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
-                       end_inset(os);
-                       eat_whitespace(p, os, context, false);
-               }
+               if (p.next_token().cat() == catEscape ||
+                   p.next_token().character() != '[') 
+                       break;
+               p.get_token(); // eat '['
+               begin_inset(os, "Argument\n");
+               os << "status collapsed\n\n";
+               parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
+               end_inset(os);
+               eat_whitespace(p, os, context, false);
+               ++optargs;
+       }
+       unsigned int reqargs = 0;
+       while (reqargs < context.layout->reqargs) {
+               eat_whitespace(p, os, context, false);
+               if (p.next_token().cat() != catBegin)
+                       break;
+               p.get_token(); // eat '{'
+               begin_inset(os, "Argument\n");
+               os << "status collapsed\n\n";
+               parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
+               end_inset(os);
+               eat_whitespace(p, os, context, false);
+               ++reqargs;
        }
        parse_text(p, os, FLAG_ITEM, outer, context);
        context.check_end_layout(os);
@@ -489,6 +614,9 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
        // We don't need really a new paragraph, but
        // we must make sure that the next item gets a \begin_layout.
        parent_context.new_paragraph(os);
+       // Set the font size to the original value. No need to output it here
+       // (Context::begin_layout() will do that if needed)
+       parent_context.font.size = size;
 }
 
 
@@ -546,12 +674,28 @@ void parse_arguments(string const & command,
                        parse_text(p, os, FLAG_ITEM, outer, context);
                        ert = "}";
                        break;
+               case item:
+                       // This argument consists only of a single item.
+                       // The presence of '{' or not must be preserved.
+                       p.skip_spaces();
+                       if (p.next_token().cat() == catBegin)
+                               ert += '{' + p.verbatim_item() + '}';
+                       else
+                               ert += p.verbatim_item();
+                       break;
                case verbatim:
                        // This argument may contain special characters
                        ert += '{' + p.verbatim_item() + '}';
                        break;
                case optional:
-                       ert += p.getOpt();
+                       // true because we must not eat whitespace
+                       // if an optional arg follows we must not strip the
+                       // brackets from this one
+                       if (i < no_arguments - 1 &&
+                           template_arguments[i+1] == optional)
+                               ert += p.getFullOpt(true);
+                       else
+                               ert += p.getOpt(true);
                        break;
                }
        }
@@ -577,65 +721,129 @@ bool parse_command(string const & command, Parser & p, ostream & os,
 
 
 /// Parses a minipage or parbox
-void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
-              Context & parent_context, bool use_parbox)
+void parse_box(Parser & p, ostream & os, unsigned outer_flags,
+               unsigned inner_flags, bool outer, Context & parent_context,
+               string const & outer_type, string const & special,
+               string const & inner_type)
 {
        string position;
        string inner_pos;
+       string hor_pos = "c";
        // We need to set the height to the LaTeX default of 1\\totalheight
        // for the case when no height argument is given
        string height_value = "1";
        string height_unit = "in";
        string height_special = "totalheight";
        string latex_height;
-       if (p.next_token().asInput() == "[") {
+       if (!inner_type.empty() && p.hasOpt()) {
                position = p.getArg('[', ']');
                if (position != "t" && position != "c" && position != "b") {
+                       cerr << "invalid position " << position << " for "
+                            << inner_type << endl;
                        position = "c";
-                       cerr << "invalid position for minipage/parbox" << endl;
                }
-               if (p.next_token().asInput() == "[") {
+               if (p.hasOpt()) {
                        latex_height = p.getArg('[', ']');
                        translate_box_len(latex_height, height_value, height_unit, height_special);
 
-                       if (p.next_token().asInput() == "[") {
+                       if (p.hasOpt()) {
                                inner_pos = p.getArg('[', ']');
                                if (inner_pos != "c" && inner_pos != "t" &&
                                    inner_pos != "b" && inner_pos != "s") {
+                                       cerr << "invalid inner_pos "
+                                            << inner_pos << " for "
+                                            << inner_type << endl;
                                        inner_pos = position;
-                                       cerr << "invalid inner_pos for minipage/parbox"
-                                            << endl;
                                }
                        }
                }
        }
        string width_value;
        string width_unit;
-       string const latex_width = p.verbatim_item();
+       string latex_width;
+       if (inner_type.empty()) {
+               if (special.empty())
+                       latex_width = "\\columnwidth";
+               else {
+                       Parser p2(special);
+                       latex_width = p2.getArg('[', ']');
+                       string const opt = p2.getArg('[', ']');
+                       if (!opt.empty()) {
+                               hor_pos = opt;
+                               if (hor_pos != "l" && hor_pos != "c" &&
+                                   hor_pos != "r") {
+                                       cerr << "invalid hor_pos " << hor_pos
+                                            << " for " << outer_type << endl;
+                                       hor_pos = "c";
+                               }
+                       }
+               }
+       } else
+               latex_width = p.verbatim_item();
        translate_len(latex_width, width_value, width_unit);
-       if (contains(width_unit, '\\') || contains(height_unit, '\\')) {
-               // LyX can't handle length variables
-               ostringstream ss;
-               if (use_parbox)
-                       ss << "\\parbox";
+       // LyX can't handle length variables
+       bool use_ert = contains(width_unit, '\\') || contains(height_unit, '\\');
+       if (!use_ert && !outer_type.empty() && !inner_type.empty()) {
+               // Look whether there is some content after the end of the
+               // inner box, but before the end of the outer box.
+               // If yes, we need to output ERT.
+               p.pushPosition();
+               if (inner_flags & FLAG_END)
+                       p.verbatimEnvironment(inner_type);
                else
-                       ss << "\\begin{minipage}";
-               if (!position.empty())
-                       ss << '[' << position << ']';
-               if (!latex_height.empty())
-                       ss << '[' << latex_height << ']';
-               if (!inner_pos.empty())
-                       ss << '[' << inner_pos << ']';
-               ss << "{" << latex_width << "}";
-               if (use_parbox)
-                       ss << '{';
+                       p.verbatim_item();
+               p.skip_spaces(true);
+               if ((outer_type == "framed" && p.next_token().asInput() != "\\end") ||
+                   (outer_type != "framed" && p.next_token().cat() != catEnd)) {
+                       // something is between the end of the inner box and
+                       // the end of the outer box, so we need to use ERT.
+                       use_ert = true;
+               }
+               p.popPosition();
+       }
+       if (use_ert) {
+               ostringstream ss;
+               if (!outer_type.empty()) {
+                       if (outer_flags & FLAG_END)
+                               ss << "\\begin{" << outer_type << '}';
+                       else {
+                               ss << '\\' << outer_type << '{';
+                               if (!special.empty())
+                                       ss << special;
+                       }
+               }
+               if (!inner_type.empty()) {
+                       if (inner_flags & FLAG_END)
+                               ss << "\\begin{" << inner_type << '}';
+                       else
+                               ss << '\\' << inner_type;
+                       if (!position.empty())
+                               ss << '[' << position << ']';
+                       if (!latex_height.empty())
+                               ss << '[' << latex_height << ']';
+                       if (!inner_pos.empty())
+                               ss << '[' << inner_pos << ']';
+                       ss << '{' << latex_width << '}';
+                       if (!(inner_flags & FLAG_END))
+                               ss << '{';
+               }
                handle_ert(os, ss.str(), parent_context);
-               parent_context.new_paragraph(os);
-               parse_text_in_inset(p, os, flags, outer, parent_context);
-               if (use_parbox)
-                       handle_ert(os, "}", parent_context);
-               else
-                       handle_ert(os, "\\end{minipage}", parent_context);
+               if (!inner_type.empty()) {
+                       parse_text(p, os, inner_flags, outer, parent_context);
+                       if (inner_flags & FLAG_END)
+                               handle_ert(os, "\\end{" + inner_type + '}',
+                                          parent_context);
+                       else
+                               handle_ert(os, "}", parent_context);
+               }
+               if (!outer_type.empty()) {
+                       parse_text(p, os, outer_flags, outer, parent_context);
+                       if (outer_flags & FLAG_END)
+                               handle_ert(os, "\\end{" + outer_type + '}',
+                                          parent_context);
+                       else
+                               handle_ert(os, "}", parent_context);
+               }
        } else {
                // LyX does not like empty positions, so we have
                // to set them to the LaTeX default values here.
@@ -644,21 +852,60 @@ void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
                if (inner_pos.empty())
                        inner_pos = position;
                parent_context.check_layout(os);
-               begin_inset(os, "Box Frameless\n");
+               begin_inset(os, "Box ");
+               if (outer_type == "framed")
+                       os << "Framed\n";
+               else if (outer_type == "framebox")
+                       os << "Boxed\n";
+               else if (outer_type == "shadowbox")
+                       os << "Shadowbox\n";
+               else if (outer_type == "shaded")
+                       os << "Shaded\n";
+               else if (outer_type == "doublebox")
+                       os << "Doublebox\n";
+               else if (outer_type.empty())
+                       os << "Frameless\n";
+               else
+                       os << outer_type << '\n';
                os << "position \"" << position << "\"\n";
-               os << "hor_pos \"c\"\n";
-               os << "has_inner_box 1\n";
+               os << "hor_pos \"" << hor_pos << "\"\n";
+               os << "has_inner_box " << !inner_type.empty() << "\n";
                os << "inner_pos \"" << inner_pos << "\"\n";
-               os << "use_parbox " << use_parbox << "\n";
+               os << "use_parbox " << (inner_type == "parbox") << '\n';
+               os << "use_makebox 0\n";
                os << "width \"" << width_value << width_unit << "\"\n";
                os << "special \"none\"\n";
                os << "height \"" << height_value << height_unit << "\"\n";
                os << "height_special \"" << height_special << "\"\n";
                os << "status open\n\n";
-               parse_text_in_inset(p, os, flags, outer, parent_context);
+               Context context(true, parent_context.textclass);
+               context.font = parent_context.font;
+
+               // FIXME, the inset layout should be plain, not standard, see bug #7846
+
+               // If we have no inner box the contens will be read with the outer box
+               if (!inner_type.empty())
+                       parse_text(p, os, inner_flags, outer, context);
+
+               // Ensure that the end of the outer box is parsed correctly:
+               // The opening brace has been eaten by parse_outer_box()
+               if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
+                       outer_flags &= ~FLAG_ITEM;
+                       outer_flags |= FLAG_BRACE_LAST;
+               }
+
+               // Find end of outer box, output contents if inner_type is
+               // empty and output possible comments
+               if (!outer_type.empty()) {
+                       // This does not output anything but comments if
+                       // inner_type is not empty (see use_ert)
+                       parse_text(p, os, outer_flags, outer, context);
+               }
+
+               context.check_end_layout(os);
                end_inset(os);
 #ifdef PRESERVE_LAYOUT
-               // lyx puts a % after the end of the minipage
+               // LyX puts a % after the end of the minipage
                if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) {
                        // new paragraph
                        //handle_comment(os, "%dummy", parent_context);
@@ -672,7 +919,8 @@ void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.skip_spaces();
                        // We add a protected space if something real follows
                        if (p.good() && p.next_token().cat() != catComment) {
-                               os << "\\InsetSpace ~\n";
+                               begin_inset(os, "space ~\n");
+                               end_inset(os);
                        }
                }
 #endif
@@ -680,6 +928,81 @@ void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
 }
 
 
+void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer,
+                     Context & parent_context, string const & outer_type,
+                     string const & special)
+{
+       eat_whitespace(p, os, parent_context, false);
+       if (flags & FLAG_ITEM) {
+               // Eat '{'
+               if (p.next_token().cat() == catBegin)
+                       p.get_token();
+               else
+                       cerr << "Warning: Ignoring missing '{' after \\"
+                            << outer_type << '.' << endl;
+               eat_whitespace(p, os, parent_context, false);
+       }
+       string inner;
+       unsigned int inner_flags = 0;
+       if (outer_type == "shaded") {
+               // These boxes never have an inner box
+               ;
+       } else if (p.next_token().asInput() == "\\parbox") {
+               inner = p.get_token().cs();
+               inner_flags = FLAG_ITEM;
+       } else if (p.next_token().asInput() == "\\begin") {
+               // Is this a minipage?
+               p.pushPosition();
+               p.get_token();
+               inner = p.getArg('{', '}');
+               p.popPosition();
+               if (inner == "minipage") {
+                       p.get_token();
+                       p.getArg('{', '}');
+                       eat_whitespace(p, os, parent_context, false);
+                       inner_flags = FLAG_END;
+               } else
+                       inner = "";
+       }
+       if (inner_flags == FLAG_END) {
+               active_environments.push_back(inner);
+               parse_box(p, os, flags, FLAG_END, outer, parent_context,
+                         outer_type, special, inner);
+               active_environments.pop_back();
+       } else {
+               parse_box(p, os, flags, inner_flags, outer, parent_context,
+                         outer_type, special, inner);
+       }
+}
+
+
+void parse_listings(Parser & p, ostream & os, Context & parent_context)
+{
+       parent_context.check_layout(os);
+       begin_inset(os, "listings\n");
+       os << "inline false\n"
+          << "status collapsed\n";
+       Context context(true, parent_context.textclass);
+       context.layout = &parent_context.textclass.plainLayout();
+       context.check_layout(os);
+       string const s = p.verbatimEnvironment("lstlisting");
+       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
+               if (*it == '\\')
+                       os << "\n\\backslash\n";
+               else if (*it == '\n') {
+                       // avoid adding an empty paragraph at the end
+                       if (it + 1 != et) {
+                               context.new_paragraph(os);
+                               context.check_layout(os);
+                       }
+               } else
+                       os << *it;
+       }
+       context.check_end_layout(os);
+       end_inset(os);
+}
+
+
 /// parse an unknown environment
 void parse_unknown_environment(Parser & p, string const & name, ostream & os,
                               unsigned flags, bool outer,
@@ -708,9 +1031,10 @@ void parse_unknown_environment(Parser & p, string const & name, ostream & os,
 
 
 void parse_environment(Parser & p, ostream & os, bool outer,
-                      Context & parent_context)
+                       string & last_env, Context & parent_context)
 {
        Layout const * newlayout;
+       InsetLayout const * newinsetlayout = 0;
        string const name = p.getArg('{', '}');
        const bool is_starred = suffixIs(name, '*');
        string const unstarred_name = rtrim(name, "*");
@@ -738,9 +1062,8 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                eat_whitespace(p, os, parent_context, false);
                parent_context.check_layout(os);
                begin_inset(os, "Float " + unstarred_name + "\n");
-               if (p.next_token().asInput() == "[") {
+               if (p.hasOpt())
                        os << "placement " << p.getArg('[', ']') << '\n';
-               }
                os << "wide " << convert<string>(is_starred)
                   << "\nsideways false"
                   << "\nstatus open\n\n";
@@ -752,9 +1075,28 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                p.skip_spaces();
        }
 
+       else if (unstarred_name == "sidewaysfigure"
+               || unstarred_name == "sidewaystable") {
+               eat_whitespace(p, os, parent_context, false);
+               parent_context.check_layout(os);
+               if (unstarred_name == "sidewaysfigure")
+                       begin_inset(os, "Float figure\n");
+               else
+                       begin_inset(os, "Float table\n");
+               os << "wide " << convert<string>(is_starred)
+                  << "\nsideways true"
+                  << "\nstatus open\n\n";
+               parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
+               end_inset(os);
+               // We don't need really a new paragraph, but
+               // we must make sure that the next item gets a \begin_layout.
+               parent_context.new_paragraph(os);
+               p.skip_spaces();
+       }
+
        else if (name == "minipage") {
                eat_whitespace(p, os, parent_context, false);
-               parse_box(p, os, FLAG_END, outer, parent_context, false);
+               parse_box(p, os, 0, FLAG_END, outer, parent_context, "", "", name);
                p.skip_spaces();
        }
 
@@ -779,23 +1121,20 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                p.skip_spaces();
        }
 
-       else if (name == "framed") {
+       else if (name == "framed" || name == "shaded") {
                eat_whitespace(p, os, parent_context, false);
-               parent_context.check_layout(os);
-               begin_inset(os, "Note Framed\n");
-               os << "status open\n";
-               parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
-               end_inset(os);
+               parse_outer_box(p, os, FLAG_END, outer, parent_context, name, "");
                p.skip_spaces();
        }
 
-       else if (name == "shaded") {
+       else if (name == "lstlisting") {
                eat_whitespace(p, os, parent_context, false);
-               parent_context.check_layout(os);
-               begin_inset(os, "Note Shaded\n");
-               os << "status open\n";
-               parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
-               end_inset(os);
+               // FIXME handle listings with parameters
+               if (p.hasOpt())
+                       parse_unknown_environment(p, name, os, FLAG_END,
+                                                 outer, parent_context);
+               else
+                       parse_listings(p, os, parent_context);
                p.skip_spaces();
        }
 
@@ -843,8 +1182,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
        }
 
        // The single '=' is meant here.
-       else if ((newlayout = findLayout(parent_context.textclass, name)) &&
-                 newlayout->isEnvironment()) {
+       else if ((newlayout = findLayout(parent_context.textclass, name, false))) {
                eat_whitespace(p, os, parent_context, false);
                Context context(true, parent_context.textclass, newlayout,
                                parent_context.layout, parent_context.font);
@@ -855,6 +1193,28 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        context.need_end_deeper = true;
                }
                parent_context.check_end_layout(os);
+               if (last_env == name) {
+                       // we need to output a separator since LyX would export
+                       // the two environments as one otherwise (bug 5716)
+                       docstring const sep = from_ascii("--Separator--");
+                       TeX2LyXDocClass const & textclass(parent_context.textclass);
+                       if (textclass.hasLayout(sep)) {
+                               Context newcontext(parent_context);
+                               newcontext.layout = &(textclass[sep]);
+                               newcontext.check_layout(os);
+                               newcontext.check_end_layout(os);
+                       } else {
+                               parent_context.check_layout(os);
+                               begin_inset(os, "Note Note\n");
+                               os << "status closed\n";
+                               Context newcontext(true, textclass,
+                                               &(textclass.defaultLayout()));
+                               newcontext.check_layout(os);
+                               newcontext.check_end_layout(os);
+                               end_inset(os);
+                               parent_context.check_end_layout(os);
+                       }
+               }
                switch (context.layout->latextype) {
                case  LATEX_LIST_ENVIRONMENT:
                        context.add_par_extra_stuff("\\labelwidthstring "
@@ -869,6 +1229,49 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        break;
                }
                context.check_deeper(os);
+               // handle known optional and required arguments
+               // layouts require all optional arguments before the required ones
+               // Unfortunately LyX can't handle arguments of list arguments (bug 7468):
+               // It is impossible to place anything after the environment name,
+               // but before the first \\item.
+               if (context.layout->latextype == LATEX_ENVIRONMENT) {
+                       bool need_layout = true;
+                       unsigned int optargs = 0;
+                       while (optargs < context.layout->optargs) {
+                               eat_whitespace(p, os, context, false);
+                               if (p.next_token().cat() == catEscape ||
+                                   p.next_token().character() != '[') 
+                                       break;
+                               p.get_token(); // eat '['
+                               if (need_layout) {
+                                       context.check_layout(os);
+                                       need_layout = false;
+                               }
+                               begin_inset(os, "Argument\n");
+                               os << "status collapsed\n\n";
+                               parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
+                               end_inset(os);
+                               eat_whitespace(p, os, context, false);
+                               ++optargs;
+                       }
+                       unsigned int reqargs = 0;
+                       while (reqargs < context.layout->reqargs) {
+                               eat_whitespace(p, os, context, false);
+                               if (p.next_token().cat() != catBegin)
+                                       break;
+                               p.get_token(); // eat '{'
+                               if (need_layout) {
+                                       context.check_layout(os);
+                                       need_layout = false;
+                               }
+                               begin_inset(os, "Argument\n");
+                               os << "status collapsed\n\n";
+                               parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
+                               end_inset(os);
+                               eat_whitespace(p, os, context, false);
+                               ++reqargs;
+                       }
+               }
                parse_text(p, os, FLAG_END, outer, context);
                context.check_end_layout(os);
                if (parent_context.deeper_paragraph) {
@@ -881,6 +1284,17 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                p.skip_spaces();
        }
 
+       // The single '=' is meant here.
+       else if ((newinsetlayout = findInsetLayout(parent_context.textclass, name, false))) {
+               eat_whitespace(p, os, parent_context, false);
+               parent_context.check_layout(os);
+               begin_inset(os, "Flex ");
+               os << to_utf8(newinsetlayout->name()) << '\n'
+                  << "status collapsed\n";
+               parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
+               end_inset(os);
+       }
+
        else if (name == "appendix") {
                // This is no good latex style, but it works and is used in some documents...
                eat_whitespace(p, os, parent_context, false);
@@ -929,6 +1343,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                parse_unknown_environment(p, name, os, FLAG_END, outer,
                                          parent_context);
 
+       last_env = name;
        active_environments.pop_back();
 }
 
@@ -1054,7 +1469,7 @@ void fix_relative_filename(string & name)
        if (FileName::isAbsolute(name))
                return;
 
-       name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFilename()),
+       name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFileName()),
                                   from_utf8(getParentFilePath())));
 }
 
@@ -1102,8 +1517,14 @@ void parse_noweb(Parser & p, ostream & os, Context & context)
                // at all.
                if (t.cat() == catEscape)
                        os << subst(t.asInput(), "\\", "\n\\backslash\n");
-               else
-                       os << subst(t.asInput(), "\n", "\n\\newline\n");
+               else {
+                       ostringstream oss;
+                       Context tmp(false, context.textclass,
+                                   &context.textclass[from_ascii("Scrap")]);
+                       tmp.need_end_layout = true;
+                       tmp.check_layout(oss);
+                       os << subst(t.asInput(), "\n", oss.str());
+               }
                // The scrap chunk is ended by an @ at the beginning of a line.
                // After the @ the line may contain a comment and/or
                // whitespace, but nothing else.
@@ -1127,6 +1548,114 @@ void parse_noweb(Parser & p, ostream & os, Context & context)
        newcontext.check_end_layout(os);
 }
 
+
+/// detects \\def, \\long\\def and \\global\\long\\def with ws and comments
+bool is_macro(Parser & p)
+{
+       Token first = p.curr_token();
+       if (first.cat() != catEscape || !p.good())
+               return false;
+       if (first.cs() == "def")
+               return true;
+       if (first.cs() != "global" && first.cs() != "long")
+               return false;
+       Token second = p.get_token();
+       int pos = 1;
+       while (p.good() && !p.isParagraph() && (second.cat() == catSpace ||
+              second.cat() == catNewline || second.cat() == catComment)) {
+               second = p.get_token();
+               pos++;
+       }
+       bool secondvalid = second.cat() == catEscape;
+       Token third;
+       bool thirdvalid = false;
+       if (p.good() && first.cs() == "global" && secondvalid &&
+           second.cs() == "long") {
+               third = p.get_token();
+               pos++;
+               while (p.good() && !p.isParagraph() &&
+                      (third.cat() == catSpace ||
+                       third.cat() == catNewline ||
+                       third.cat() == catComment)) {
+                       third = p.get_token();
+                       pos++;
+               }
+               thirdvalid = third.cat() == catEscape;
+       }
+       for (int i = 0; i < pos; ++i)
+               p.putback();
+       if (!secondvalid)
+               return false;
+       if (!thirdvalid)
+               return (first.cs() == "global" || first.cs() == "long") &&
+                      second.cs() == "def";
+       return first.cs() == "global" && second.cs() == "long" &&
+              third.cs() == "def";
+}
+
+
+/// Parse a macro definition (assumes that is_macro() returned true)
+void parse_macro(Parser & p, ostream & os, Context & context)
+{
+       context.check_layout(os);
+       Token first = p.curr_token();
+       Token second;
+       Token third;
+       string command = first.asInput();
+       if (first.cs() != "def") {
+               p.get_token();
+               eat_whitespace(p, os, context, false);
+               second = p.curr_token();
+               command += second.asInput();
+               if (second.cs() != "def") {
+                       p.get_token();
+                       eat_whitespace(p, os, context, false);
+                       third = p.curr_token();
+                       command += third.asInput();
+               }
+       }
+       eat_whitespace(p, os, context, false);
+       string const name = p.get_token().cs();
+       eat_whitespace(p, os, context, false);
+
+       // parameter text
+       bool simple = true;
+       string paramtext;
+       int arity = 0;
+       while (p.next_token().cat() != catBegin) {
+               if (p.next_token().cat() == catParameter) {
+                       // # found
+                       p.get_token();
+                       paramtext += "#";
+
+                       // followed by number?
+                       if (p.next_token().cat() == catOther) {
+                               char c = p.getChar();
+                               paramtext += c;
+                               // number = current arity + 1?
+                               if (c == arity + '0' + 1)
+                                       ++arity;
+                               else
+                                       simple = false;
+                       } else
+                               paramtext += p.get_token().cs();
+               } else {
+                       paramtext += p.get_token().cs();
+                       simple = false;
+               }
+       }
+
+       // only output simple (i.e. compatible) macro as FormulaMacros
+       string ert = '\\' + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
+       if (simple) {
+               context.check_layout(os);
+               begin_inset(os, "FormulaMacro");
+               os << "\n\\def" << ert;
+               end_inset(os);
+       } else
+               handle_ert(os, command + ert, context);
+}
+
 } // anonymous namespace
 
 
@@ -1134,17 +1663,19 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                Context & context)
 {
        Layout const * newlayout = 0;
-       // store the current selectlanguage to be used after \foreignlanguage
-       string selectlang;
-       // Store the latest bibliographystyle (needed for bibtex inset)
+       InsetLayout const * newinsetlayout = 0;
+       // Store the latest bibliographystyle and nocite{*} option
+       // (needed for bibtex inset)
+       string btprint;
        string bibliographystyle;
        bool const use_natbib = used_packages.find("natbib") != used_packages.end();
        bool const use_jurabib = used_packages.find("jurabib") != used_packages.end();
+       string last_env;
        while (p.good()) {
                Token const & t = p.get_token();
 
 #ifdef FILEDEBUG
-               cerr << "t: " << t << " flags: " << flags << "\n";
+               debugToken(cerr, t, flags);
 #endif
 
                if (flags & FLAG_ITEM) {
@@ -1163,8 +1694,17 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        flags |= FLAG_LEAVE;
                }
 
-               if (t.character() == ']' && (flags & FLAG_BRACK_LAST))
+               if (t.cat() != catEscape && t.character() == ']' &&
+                   (flags & FLAG_BRACK_LAST))
                        return;
+               if (t.cat() == catEnd && (flags & FLAG_BRACE_LAST))
+                       return;
+
+               // If there is anything between \end{env} and \begin{env} we
+               // don't need to output a separator.
+               if (t.cat() != catSpace && t.cat() != catNewline &&
+                   t.asInput() != "\\begin")
+                       last_env = "";
 
                //
                // cat codes
@@ -1254,8 +1794,33 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        handle_ert(os, s, context);
                }
 
-               else if (t.cat() == catLetter ||
-                              t.cat() == catOther ||
+               else if (t.cat() == catLetter) {
+                       context.check_layout(os);
+                       // Workaround for bug 4752.
+                       // FIXME: This whole code block needs to be removed
+                       //        when the bug is fixed and tex2lyx produces
+                       //        the updated file format.
+                       // The replacement algorithm in LyX is so stupid that
+                       // it even translates a phrase if it is part of a word.
+                       bool handled = false;
+                       for (int const * l = known_phrase_lengths; *l; ++l) {
+                               string phrase = t.cs();
+                               for (int i = 1; i < *l && p.next_token().isAlnumASCII(); ++i)
+                                       phrase += p.get_token().cs();
+                               if (is_known(phrase, known_coded_phrases)) {
+                                       handle_ert(os, phrase, context);
+                                       handled = true;
+                                       break;
+                               } else {
+                                       for (size_t i = 1; i < phrase.length(); ++i)
+                                               p.putback();
+                               }
+                       }
+                       if (!handled)
+                               os << t.cs();
+               }
+
+               else if (t.cat() == catOther ||
                               t.cat() == catAlign ||
                               t.cat() == catParameter) {
                        // This translates "&" to "\\&" which may be wrong...
@@ -1276,8 +1841,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        if (t.character() == '~') {
                                if (context.layout->free_spacing)
                                        os << ' ';
-                               else
-                                       os << "\\InsetSpace ~\n";
+                               else {
+                                       begin_inset(os, "space ~\n");
+                                       end_inset(os);
+                               }
                        } else
                                os << t.cs();
                }
@@ -1410,7 +1977,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                }
 
                else if (t.cs() == "begin")
-                       parse_environment(p, os, outer, context);
+                       parse_environment(p, os, outer, last_env, context);
 
                else if (t.cs() == "end") {
                        if (flags & FLAG_END) {
@@ -1428,7 +1995,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.skip_spaces();
                        string s;
                        bool optarg = false;
-                       if (p.next_token().character() == '[') {
+                       if (p.next_token().cat() != catEscape &&
+                           p.next_token().character() == '[') {
                                p.get_token(); // eat '['
                                s = parse_text_snippet(p, FLAG_BRACK_LAST,
                                                       outer, context);
@@ -1447,7 +2015,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                        if (optarg) {
                                if (context.layout->labeltype != LABEL_MANUAL) {
-                                       // lyx does not support \item[\mybullet]
+                                       // LyX does not support \item[\mybullet]
                                        // in itemize environments
                                        handle_ert(os, "[", context);
                                        os << s;
@@ -1464,56 +2032,24 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "bibitem") {
                        context.set_item();
                        context.check_layout(os);
-                       begin_command_inset(os, "bibitem", "bibitem");
-                       os << p.getOpt();
-                       os << "key " << '"' << p.verbatim_item() << '"' << "\n";
-                       end_inset(os);
-               }
-
-               else if (t.cs() == "def") {
-                       context.check_layout(os);
-                       eat_whitespace(p, os, context, false);
-                       string name = p.get_token().cs();
-                       eat_whitespace(p, os, context, false);
-
-                       // parameter text
-                       bool simple = true;
-                       string paramtext;
-                       int arity = 0;
-                       while (p.next_token().cat() != catBegin) {
-                               if (p.next_token().cat() == catParameter) {
-                                       // # found
-                                       p.get_token();
-                                       paramtext += "#";
-
-                                       // followed by number?
-                                       if (p.next_token().cat() == catOther) {
-                                               char c = p.getChar();
-                                               paramtext += c;
-                                               // number = current arity + 1?
-                                               if (c == arity + '0' + 1)
-                                                       ++arity;
-                                               else
-                                                       simple = false;
-                                       } else
-                                               paramtext += p.get_token().asString();
-                               } else {
-                                       paramtext += p.get_token().asString();
-                                       simple = false;
-                               }
-                       }
-
-                       // only output simple (i.e. compatible) macro as FormulaMacros
-                       string ert = "\\def\\" + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
-                       if (simple) {
-                               context.check_layout(os);
-                               begin_inset(os, "FormulaMacro");
-                               os << "\n" << ert;
+                       string label = convert_command_inset_arg(p.getArg('[', ']'));
+                       string key = convert_command_inset_arg(p.verbatim_item());
+                       if (contains(label, '\\') || contains(key, '\\')) {
+                               // LyX can't handle LaTeX commands in labels or keys
+                               handle_ert(os, t.asInput() + '[' + label +
+                                              "]{" + p.verbatim_item() + '}',
+                                          context);
+                       } else {
+                               begin_command_inset(os, "bibitem", "bibitem");
+                               os << "label \"" << label << "\"\n"
+                                     "key \"" << key << "\"\n";
                                end_inset(os);
-                       } else
-                               handle_ert(os, ert, context);
+                       }
                }
 
+               else if (is_macro(p))
+                       parse_macro(p, os, context);
+
                else if (t.cs() == "noindent") {
                        p.skip_spaces();
                        context.add_par_extra_stuff("\\noindent\n");
@@ -1546,58 +2082,56 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                // Must attempt to parse "Section*" before "Section".
                else if ((p.next_token().asInput() == "*") &&
                         context.new_layout_allowed &&
-                        (newlayout = findLayout(context.textclass, t.cs() + '*')) &&
-                        newlayout->isCommand()) {
-                       TeXFont const oldFont = context.font;
-                       // save the current font size
-                       string const size = oldFont.size;
-                       // reset the font size to default, because the
-                       // font size switches don't affect section
-                       // headings and the like
-                       context.font.size = known_coded_sizes[0];
-                       output_font_change(os, oldFont, context.font);
+                        (newlayout = findLayout(context.textclass, t.cs() + '*', true))) {
                        // write the layout
                        p.get_token();
                        output_command_layout(os, p, outer, context, newlayout);
-                       // set the font size to the original value
-                       context.font.size = size;
-                       output_font_change(os, oldFont, context.font);
                        p.skip_spaces();
                }
 
                // Section headings and the like
                else if (context.new_layout_allowed &&
-                        (newlayout = findLayout(context.textclass, t.cs())) &&
-                        newlayout->isCommand()) {
-                       TeXFont const oldFont = context.font;
-                       // save the current font size
-                       string const size = oldFont.size;
-                       // reset the font size to default, because the font size switches don't
-                       // affect section headings and the like
-                       context.font.size = known_coded_sizes[0];
-                       output_font_change(os, oldFont, context.font);
+                        (newlayout = findLayout(context.textclass, t.cs(), true))) {
                        // write the layout
                        output_command_layout(os, p, outer, context, newlayout);
-                       // set the font size to the original value
-                       context.font.size = size;
-                       output_font_change(os, oldFont, context.font);
                        p.skip_spaces();
                }
 
-               // Special handling for \caption
-               // FIXME: remove this when InsetCaption is supported.
-               else if (context.new_layout_allowed &&
-                        t.cs() == captionlayout()->latexname()) {
-                       output_command_layout(os, p, outer, context, 
-                                             captionlayout());
+               else if (t.cs() == "caption") {
+                       p.skip_spaces();
+                       context.check_layout(os);
+                       p.skip_spaces();
+                       begin_inset(os, "Caption\n\n");
+                       Context newcontext(true, context.textclass);
+                       newcontext.font = context.font;
+                       newcontext.check_layout(os);
+                       if (p.next_token().cat() != catEscape &&
+                           p.next_token().character() == '[') {
+                               p.get_token(); // eat '['
+                               begin_inset(os, "Argument\n");
+                               os << "status collapsed\n";
+                               parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
+                               end_inset(os);
+                               eat_whitespace(p, os, context, false);
+                       }
+                       parse_text(p, os, FLAG_ITEM, outer, context);
+                       context.check_end_layout(os);
+                       // We don't need really a new paragraph, but
+                       // we must make sure that the next item gets a \begin_layout.
+                       context.new_paragraph(os);
+                       end_inset(os);
                        p.skip_spaces();
+                       newcontext.check_end_layout(os);
                }
 
                else if (t.cs() == "includegraphics") {
                        bool const clip = p.next_token().asInput() == "*";
                        if (clip)
                                p.get_token();
-                       map<string, string> opts = split_map(p.getArg('[', ']'));
+                       string const arg = p.getArg('[', ']');
+                       map<string, string> opts;
+                       vector<string> keys;
+                       split_map(arg, opts, keys);
                        if (clip)
                                opts["clip"] = string();
                        string name = normalize_filename(p.verbatim_item());
@@ -1631,8 +2165,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                     << endl;
                                        }
                                        name = dvips_name;
-                               } else if (!pdftex_name.empty())
+                               } else if (!pdftex_name.empty()) {
                                        name = pdftex_name;
+                                       pdflatex = true;
+                               }
                        }
 
                        if (makeAbsPath(name, path).exists())
@@ -1657,9 +2193,20 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                val = val*100;
                                os << "\tscale " << val << '\n';
                        }
-                       if (opts.find("angle") != opts.end())
+                       if (opts.find("angle") != opts.end()) {
                                os << "\trotateAngle "
                                   << opts["angle"] << '\n';
+                               vector<string>::const_iterator a =
+                                       find(keys.begin(), keys.end(), "angle");
+                               vector<string>::const_iterator s =
+                                       find(keys.begin(), keys.end(), "width");
+                               if (s == keys.end())
+                                       s = find(keys.begin(), keys.end(), "height");
+                               if (s == keys.end())
+                                       s = find(keys.begin(), keys.end(), "scale");
+                               if (s != keys.end() && distance(s, a) > 0)
+                                       os << "\tscaleBeforeRotation\n";
+                       }
                        if (opts.find("origin") != opts.end()) {
                                ostringstream ss;
                                string const opt = opts["origin"];
@@ -1767,47 +2314,37 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                           context);
                }
 
-               else if (t.cs() == "hfill") {
-                       context.check_layout(os);
-                       os << "\n\\hfill\n";
-                       skip_braces(p);
-                       p.skip_spaces();
-               }
-
                else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
                        // FIXME: Somehow prevent title layouts if
                        // "maketitle" was not found
-                       p.skip_spaces();
-                       skip_braces(p); // swallow this
+                       // swallow this
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "tableofcontents") {
-                       p.skip_spaces();
                        context.check_layout(os);
                        begin_command_inset(os, "toc", "tableofcontents");
                        end_inset(os);
-                       skip_braces(p); // swallow this
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "listoffigures") {
-                       p.skip_spaces();
                        context.check_layout(os);
                        begin_inset(os, "FloatList figure\n");
                        end_inset(os);
-                       skip_braces(p); // swallow this
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "listoftables") {
-                       p.skip_spaces();
                        context.check_layout(os);
                        begin_inset(os, "FloatList table\n");
                        end_inset(os);
-                       skip_braces(p); // swallow this
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "listof") {
                        p.skip_spaces(true);
-                       string const name = p.get_token().asString();
+                       string const name = p.get_token().cs();
                        if (context.textclass.floats().typeExist(name)) {
                                context.check_layout(os);
                                begin_inset(os, "FloatList ");
@@ -1896,7 +2433,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                handle_ert(os, t.asInput() + "{" + color + "}", context);
                }
 
-               else if (t.cs() == "underbar") {
+               else if (t.cs() == "underbar" || t.cs() == "uline") {
+                       // \underbar is not 100% correct (LyX outputs \uline
+                       // of ulem.sty). The difference is that \ulem allows
+                       // line breaks, and \underbar does not.
                        // Do NOT handle \underline.
                        // \underbar cuts through y, g, q, p etc.,
                        // \underline does not.
@@ -1907,7 +2447,16 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        os << "\n\\bar default\n";
                }
 
-               else if (t.cs() == "emph" || t.cs() == "noun") {
+               else if (t.cs() == "sout") {
+                       context.check_layout(os);
+                       os << "\n\\strikeout on\n";
+                       parse_text_snippet(p, os, FLAG_ITEM, outer, context);
+                       context.check_layout(os);
+                       os << "\n\\strikeout default\n";
+               }
+
+               else if (t.cs() == "uuline" || t.cs() == "uwave" ||
+                        t.cs() == "emph" || t.cs() == "noun") {
                        context.check_layout(os);
                        os << "\n\\" << t.cs() << " on\n";
                        parse_text_snippet(p, os, FLAG_ITEM, outer, context);
@@ -1915,21 +2464,99 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        os << "\n\\" << t.cs() << " default\n";
                }
 
+               // FIXME, the inset layout should be plain, not standard, see bug #7846
+               else if (t.cs() == "phantom" || t.cs() == "hphantom" ||
+                            t.cs() == "vphantom") {
+                       context.check_layout(os);
+                       if (t.cs() == "phantom")
+                               begin_inset(os, "Phantom Phantom\n");
+                       if (t.cs() == "hphantom")
+                               begin_inset(os, "Phantom Hhantom\n");
+                       if (t.cs() == "vphantom")
+                               begin_inset(os, "Phantom Vhantom\n");
+                       os << "status open\n";
+                       parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
+                       end_inset(os);
+               }
+               
                else if (t.cs() == "lyxline") {
+                       // swallow size argument (it is not used anyway)
+                       p.getArg('{', '}');
+                       if (!context.atParagraphStart()) {
+                               // so our line is in the middle of a paragraph
+                               // we need to add a new line, lest this line
+                               // follow the other content on that line and
+                               // run off the side of the page
+                               // FIXME: This may create an empty paragraph,
+                               //        but without that it would not be
+                               //        possible to set noindent below.
+                               //        Fortunately LaTeX does not care
+                               //        about the empty paragraph.
+                               context.new_paragraph(os);
+                       }
+                       if (h_paragraph_separation == "indent") {
+                               // we need to unindent, lest the line be too long
+                               context.add_par_extra_stuff("\\noindent\n");
+                       }
                        context.check_layout(os);
-                       os << "\\lyxline";
+                       begin_command_inset(os, "line", "rule");
+                       os << "offset \"0.5ex\"\n"
+                             "width \"100line%\"\n"
+                             "height \"1pt\"\n";
+                       end_inset(os);
                }
 
-               else if (is_known(t.cs(), known_ref_commands)) {
-                       context.check_layout(os);
-                       begin_command_inset(os, "ref", t.cs());
-                       // lyx cannot handle newlines in a latex command
-                       // FIXME: Move the substitution into parser::getOpt()?
-                       os << subst(p.getOpt(), "\n", " ");
-                       os << "reference " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+               else if (t.cs() == "rule") {
+                       string offset = "";
+                       string width;
+                       string thickness;
+                       if (p.hasOpt())
+                               offset = p.getArg('[', ']');
+                       width = p.getArg('{', '}');
+                       thickness = p.getArg('{', '}');
+                                       
+                       context.check_layout(os);
+                       begin_command_inset(os, "line", "rule");
+                       if (!offset.empty())
+                               os << "offset \"" << translate_len(offset) << "\"\n";
+                       os << "width \"" << translate_len(width) << "\"\n"
+                                 "height \"" << translate_len(thickness) << "\"\n";
                        end_inset(os);
                }
 
+               else if (is_known(t.cs(), known_phrases) ||
+                        (t.cs() == "protect" &&
+                         p.next_token().cat() == catEscape &&
+                         is_known(p.next_token().cs(), known_phrases))) {
+                       // LyX sometimes puts a \protect in front, so we have to ignore it
+                       // FIXME: This needs to be changed when bug 4752 is fixed.
+                       char const * const * where = is_known(
+                               t.cs() == "protect" ? p.get_token().cs() : t.cs(),
+                               known_phrases);
+                       context.check_layout(os);
+                       os << known_coded_phrases[where - known_phrases];
+                       skip_spaces_braces(p);
+               }
+
+               else if (is_known(t.cs(), known_ref_commands)) {
+                       string const opt = p.getOpt();
+                       if (opt.empty()) {
+                               context.check_layout(os);
+                               char const * const * where = is_known(t.cs(),
+                                       known_ref_commands);
+                               begin_command_inset(os, "ref",
+                                       known_coded_ref_commands[where - known_ref_commands]);
+                               os << "reference \""
+                                  << convert_command_inset_arg(p.verbatim_item())
+                                  << "\"\n";
+                               end_inset(os);
+                       } else {
+                               // LyX does not support optional arguments of ref commands
+                               handle_ert(os, t.asInput() + '[' + opt + "]{" +
+                                              p.verbatim_item() + "}", context);
+                       }
+               }
+
                else if (use_natbib &&
                         is_known(t.cs(), known_natbib_commands) &&
                         ((t.cs() != "citefullauthor" &&
@@ -1973,22 +2600,31 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        if (!after.empty()) {
                                after.erase(0, 1);
                                after.erase(after.length() - 1, 1);
+                               after = convert_command_inset_arg(after);
                        }
                        if (!before.empty()) {
                                before.erase(0, 1);
                                before.erase(before.length() - 1, 1);
+                               before = convert_command_inset_arg(before);
                        }
                        begin_command_inset(os, "citation", command);
                        os << "after " << '"' << after << '"' << "\n";
                        os << "before " << '"' << before << '"' << "\n";
-                       os << "key " << '"' << p.verbatim_item() << '"' << "\n";
+                       os << "key \""
+                          << convert_command_inset_arg(p.verbatim_item())
+                          << "\"\n";
                        end_inset(os);
                }
 
                else if (use_jurabib &&
-                        is_known(t.cs(), known_jurabib_commands)) {
+                        is_known(t.cs(), known_jurabib_commands) &&
+                        (t.cs() == "cite" || p.next_token().asInput() != "*")) {
                        context.check_layout(os);
-                       string const command = t.cs();
+                       string command = t.cs();
+                       if (p.next_token().asInput() == "*") {
+                               command += '*';
+                               p.get_token();
+                       }
                        char argumentOrder = '\0';
                        vector<string> const & options = used_packages["jurabib"];
                        if (find(options.begin(), options.end(),
@@ -2029,50 +2665,92 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        end_inset(os);
                }
 
-               else if (t.cs() == "cite") {
+               else if (t.cs() == "cite"
+                       || t.cs() == "nocite") {
                        context.check_layout(os);
-                       // lyx cannot handle newlines in a latex command
-                       string after = subst(p.getOpt(), "\n", " ");
-                       if (!after.empty()) {
-                               after.erase(0, 1);
-                               after.erase(after.length() - 1, 1);
-                       }
-                       begin_command_inset(os, "citation", t.cs());
-                       os << "after " << '"' << after << '"' << "\n";
-                       os << "key " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
-                       end_inset(os);
+                       string after = convert_command_inset_arg(p.getArg('[', ']'));
+                       string key = convert_command_inset_arg(p.verbatim_item());
+                       // store the case that it is "\nocite{*}" to use it later for
+                       // the BibTeX inset
+                       if (key != "*") {
+                               begin_command_inset(os, "citation", t.cs());
+                               os << "after " << '"' << after << '"' << "\n";
+                               os << "key " << '"' << key << '"' << "\n";
+                               end_inset(os);
+                       } else if (t.cs() == "nocite")
+                               btprint = key;
                }
 
                else if (t.cs() == "index") {
                        context.check_layout(os);
-                       begin_inset(os, "LatexCommand ");
-                       os << t.cs() << "\n";;
-                       // lyx cannot handle newlines in a latex command
-                       os << "name " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       begin_inset(os, "Index idx\n");
+                       os << "status collapsed\n";
+                       parse_text_in_inset(p, os, FLAG_ITEM, false, context, "Index");
+                       end_inset(os);
+               }
+
+               else if (t.cs() == "nomenclature") {
+                       context.check_layout(os);
+                       begin_command_inset(os, "nomenclature", "nomenclature");
+                       string prefix = convert_command_inset_arg(p.getArg('[', ']'));
+                       if (!prefix.empty())
+                               os << "prefix " << '"' << prefix << '"' << "\n";
+                       os << "symbol " << '"'
+                          << convert_command_inset_arg(p.verbatim_item());
+                       os << "\"\ndescription \""
+                          << convert_command_inset_arg(p.verbatim_item())
+                          << "\"\n";
                        end_inset(os);
                }
                
                else if (t.cs() == "label") {
                        context.check_layout(os);
-                       begin_command_inset(os, t.cs(), t.cs());
-                       // lyx cannot handle newlines in a latex command
-                       os << "name " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       begin_command_inset(os, "label", "label");
+                       os << "name \""
+                          << convert_command_inset_arg(p.verbatim_item())
+                          << "\"\n";
                        end_inset(os);
                }
 
                else if (t.cs() == "printindex") {
                        context.check_layout(os);
-                       begin_command_inset(os, "index_print", t.cs());
+                       begin_command_inset(os, "index_print", "printindex");
+                       os << "type \"idx\"\n";
                        end_inset(os);
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
-               else if (t.cs() == "url") {
+               else if (t.cs() == "printnomenclature") {
+                       string width = "";
+                       string width_type = "";
                        context.check_layout(os);
-                       begin_inset(os, "LatexCommand ");
-                       os << t.cs() << "\n";;
-                       // lyx cannot handle newlines in a latex command
-                       os << "target " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       begin_command_inset(os, "nomencl_print", "printnomenclature");
+                       // case of a custom width
+                       if (p.hasOpt()) {
+                               width = p.getArg('[', ']');
+                               width = translate_len(width);
+                               width_type = "custom";
+                       }
+                       // case of no custom width
+                       // the case of no custom width but the width set
+                       // via \settowidth{\nomlabelwidth}{***} cannot be supported
+                       // because the user could have set anything, not only the width
+                       // of the longest label (which would be width_type = "auto")
+                       string label = convert_command_inset_arg(p.getArg('{', '}'));
+                       if (label.empty() && width_type.empty())
+                               width_type = "none";
+                       os << "set_width \"" << width_type << "\"\n";
+                       if (width_type == "custom")
+                               os << "width \"" << width << '\"';
+                       end_inset(os);
+                       skip_spaces_braces(p);
+               }
+
+               else if ((t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
+                       context.check_layout(os);
+                       begin_inset(os, "script ");
+                       os << t.cs().substr(4) << '\n';
+                       parse_text_in_inset(p, os, FLAG_ITEM, false, context);
                        end_inset(os);
                }
 
@@ -2094,9 +2772,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        char const * const * where = is_known(t.cs(), known_sizes);
                        context.check_layout(os);
                        TeXFont const oldFont = context.font;
-                       // the font size index differs by 1, because the known_coded_sizes
-                       // has additionally a "default" entry
-                       context.font.size = known_coded_sizes[where - known_sizes + 1];
+                       context.font.size = known_coded_sizes[where - known_sizes];
                        output_font_change(os, oldFont, context.font);
                        eat_whitespace(p, os, context, false);
                }
@@ -2183,21 +2859,15 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // save the language for the case that a
                        // \foreignlanguage is used 
 
-                       //FIXME: this is wrong, the language should
-                       // be saved in the context. (JMarc)
-                       selectlang = subst(p.verbatim_item(), "\n", " ");
-                       os << "\\lang " << selectlang << "\n";
+                       context.font.language = babel2lyx(p.verbatim_item());
+                       os << "\n\\lang " << context.font.language << "\n";
                }
 
                else if (t.cs() == "foreignlanguage") {
-                       context.check_layout(os);
-                       os << "\n\\lang " << subst(p.verbatim_item(), "\n", " ") << "\n";
-                       os << subst(p.verbatim_item(), "\n", " ");
-                       // FIXME: the second argument of selectlanguage
-                       // has to be parsed (like for \textsf, for
-                       // example). 
-                       // set back to last selectlanguage
-                       os << "\n\\lang " << selectlang << "\n";
+                       string const lang = babel2lyx(p.verbatim_item());
+                       parse_text_attributes(p, os, FLAG_ITEM, outer,
+                                             context, "\\lang",
+                                             context.font.language, lang);
                }
 
                else if (t.cs() == "inputencoding") {
@@ -2205,34 +2875,40 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        string const enc = subst(p.verbatim_item(), "\n", " ");
                        p.setEncoding(enc);
                }
-               else if (t.cs() == "LyX" || t.cs() == "TeX"
-                        || t.cs() == "LaTeX") {
-                       context.check_layout(os);
-                       os << t.cs();
-                       skip_braces(p); // eat {}
-               }
-
-               else if (t.cs() == "LaTeXe") {
-                       context.check_layout(os);
-                       os << "LaTeX2e";
-                       skip_braces(p); // eat {}
-               }
 
                else if (t.cs() == "ldots") {
                        context.check_layout(os);
-                       skip_braces(p);
                        os << "\\SpecialChar \\ldots{}\n";
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "lyxarrow") {
                        context.check_layout(os);
                        os << "\\SpecialChar \\menuseparator\n";
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "textcompwordmark") {
                        context.check_layout(os);
                        os << "\\SpecialChar \\textcompwordmark{}\n";
+                       skip_spaces_braces(p);
+               }
+
+               else if (t.cs() == "slash") {
+                       context.check_layout(os);
+                       os << "\\SpecialChar \\slash{}\n";
+                       skip_spaces_braces(p);
+               }
+
+               else if (t.cs() == "nobreakdash" && p.next_token().asInput() == "-") {
+                       context.check_layout(os);
+                       os << "\\SpecialChar \\nobreakdash-\n";
+                       p.get_token();
+               }
+
+               else if (t.cs() == "textquotedbl") {
+                       context.check_layout(os);
+                       os << "\"";
                        skip_braces(p);
                }
 
@@ -2250,19 +2926,19 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "textasciitilde") {
                        context.check_layout(os);
                        os << '~';
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "textasciicircum") {
                        context.check_layout(os);
                        os << '^';
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "textbackslash") {
                        context.check_layout(os);
                        os << "\n\\backslash\n";
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#"
@@ -2310,7 +2986,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // try to see whether the string is in unicodesymbols
                        docstring rem;
                        string command = t.asInput() + "{" 
-                               + trim(p.verbatim_item())
+                               + trimSpaceAndEol(p.verbatim_item())
                                + "}";
                        docstring s = encodings.fromLaTeXCommand(from_utf8(command), rem);
                        if (!s.empty()) {
@@ -2326,33 +3002,37 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                else if (t.cs() == "\\") {
                        context.check_layout(os);
-                       string const next = p.next_token().asInput();
-                       if (next == "[")
+                       if (p.hasOpt())
                                handle_ert(os, "\\\\" + p.getOpt(), context);
-                       else if (next == "*") {
+                       else if (p.next_token().asInput() == "*") {
                                p.get_token();
+                               // getOpt() eats the following space if there
+                               // is no optional argument, but that is OK
+                               // here since it has no effect in the output.
                                handle_ert(os, "\\\\*" + p.getOpt(), context);
                        }
                        else {
-                               os << "\n\\newline\n";
+                               begin_inset(os, "Newline newline");
+                               end_inset(os);
                        }
                }
 
                else if (t.cs() == "newline" ||
-                       t.cs() == "linebreak") {
+                        (t.cs() == "linebreak" && !p.hasOpt())) {
                        context.check_layout(os);
-                       os << "\n\\" << t.cs() << "\n";
-                       skip_braces(p); // eat {}
+                       begin_inset(os, "Newline ");
+                       os << t.cs();
+                       end_inset(os);
+                       skip_spaces_braces(p);
                }
 
                else if (t.cs() == "input" || t.cs() == "include"
                         || t.cs() == "verbatiminput") {
-                       string name = '\\' + t.cs();
+                       string name = t.cs();
                        if (t.cs() == "verbatiminput"
                            && p.next_token().asInput() == "*")
                                name += p.get_token().asInput();
                        context.check_layout(os);
-                       begin_inset(os, "Include ");
                        string filename(normalize_filename(p.getArg('{', '}')));
                        string const path = getMasterFilePath();
                        // We want to preserve relative / absolute filenames,
@@ -2367,41 +3047,129 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                if (!tex_name.empty())
                                        filename = tex_name;
                        }
+                       bool external = false;
+                       string outname;
                        if (makeAbsPath(filename, path).exists()) {
                                string const abstexname =
-                                       makeAbsPath(filename, path).absFilename();
+                                       makeAbsPath(filename, path).absFileName();
                                string const abslyxname =
                                        changeExtension(abstexname, ".lyx");
+                               string const absfigname =
+                                       changeExtension(abstexname, ".fig");
                                fix_relative_filename(filename);
                                string const lyxname =
                                        changeExtension(filename, ".lyx");
-                               if (t.cs() != "verbatiminput" &&
+                               bool xfig = false;
+                               external = FileName(absfigname).exists();
+                               if (t.cs() == "input") {
+                                       string const ext = getExtension(abstexname);
+
+                                       // Combined PS/LaTeX:
+                                       // x.eps, x.pstex_t (old xfig)
+                                       // x.pstex, x.pstex_t (new xfig, e.g. 3.2.5)
+                                       FileName const absepsname(
+                                               changeExtension(abstexname, ".eps"));
+                                       FileName const abspstexname(
+                                               changeExtension(abstexname, ".pstex"));
+                                       bool const xfigeps =
+                                               (absepsname.exists() ||
+                                                abspstexname.exists()) &&
+                                               ext == "pstex_t";
+
+                                       // Combined PDF/LaTeX:
+                                       // x.pdf, x.pdftex_t (old xfig)
+                                       // x.pdf, x.pdf_t (new xfig, e.g. 3.2.5)
+                                       FileName const abspdfname(
+                                               changeExtension(abstexname, ".pdf"));
+                                       bool const xfigpdf =
+                                               abspdfname.exists() &&
+                                               (ext == "pdftex_t" || ext == "pdf_t");
+                                       if (xfigpdf)
+                                               pdflatex = true;
+
+                                       // Combined PS/PDF/LaTeX:
+                                       // x_pspdftex.eps, x_pspdftex.pdf, x.pspdftex
+                                       string const absbase2(
+                                               removeExtension(abstexname) + "_pspdftex");
+                                       FileName const abseps2name(
+                                               addExtension(absbase2, ".eps"));
+                                       FileName const abspdf2name(
+                                               addExtension(absbase2, ".pdf"));
+                                       bool const xfigboth =
+                                               abspdf2name.exists() &&
+                                               abseps2name.exists() && ext == "pspdftex";
+
+                                       xfig = xfigpdf || xfigeps || xfigboth;
+                                       external = external && xfig;
+                               }
+                               if (external) {
+                                       outname = changeExtension(filename, ".fig");
+                               } else if (xfig) {
+                                       // Don't try to convert, the result
+                                       // would be full of ERT.
+                                       outname = filename;
+                               } else if (t.cs() != "verbatiminput" &&
                                    tex2lyx(abstexname, FileName(abslyxname),
                                            p.getEncoding())) {
-                                       os << name << '{' << lyxname << "}\n";
+                                       outname = lyxname;
                                } else {
-                                       os << name << '{' << filename << "}\n";
+                                       outname = filename;
                                }
                        } else {
                                cerr << "Warning: Could not find included file '"
                                     << filename << "'." << endl;
-                               os << name << '{' << filename << "}\n";
+                               outname = filename;
+                       }
+                       if (external) {
+                               begin_inset(os, "External\n");
+                               os << "\ttemplate XFig\n"
+                                  << "\tfilename " << outname << '\n';
+                       } else {
+                               begin_command_inset(os, "include", name);
+                               os << "preview false\n"
+                                     "filename \"" << outname << "\"\n";
                        }
-                       os << "preview false\n";
                        end_inset(os);
                }
 
                else if (t.cs() == "bibliographystyle") {
                        // store new bibliographystyle
                        bibliographystyle = p.verbatim_item();
-                       // output new bibliographystyle.
-                       // This is only necessary if used in some other macro than \bibliography.
-                       handle_ert(os, "\\bibliographystyle{" + bibliographystyle + "}", context);
+                       // If any other command than \bibliography and
+                       // \nocite{*} follows, we need to output the style
+                       // (because it might be used by that command).
+                       // Otherwise, it will automatically be output by LyX.
+                       p.pushPosition();
+                       bool output = true;
+                       for (Token t2 = p.get_token(); p.good(); t2 = p.get_token()) {
+                               if (t2.cat() == catBegin)
+                                       break;
+                               if (t2.cat() != catEscape)
+                                       continue;
+                               if (t2.cs() == "nocite") {
+                                       if (p.getArg('{', '}') == "*")
+                                               continue;
+                               } else if (t2.cs() == "bibliography")
+                                       output = false;
+                               break;
+                       }
+                       p.popPosition();
+                       if (output) {
+                               handle_ert(os,
+                                       "\\bibliographystyle{" + bibliographystyle + '}',
+                                       context);
+                       }
                }
 
                else if (t.cs() == "bibliography") {
                        context.check_layout(os);
                        begin_command_inset(os, "bibtex", "bibtex");
+                       if (!btprint.empty()) {
+                               os << "btprint " << '"' << "btPrintAll" << '"' << "\n";
+                               // clear the string because the next BibTeX inset can be without the
+                               // \nocite{*} option
+                               btprint.clear();
+                       }
                        os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n";
                        // Do we have a bibliographystyle set?
                        if (!bibliographystyle.empty())
@@ -2410,8 +3178,18 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                }
 
                else if (t.cs() == "parbox")
-                       parse_box(p, os, FLAG_ITEM, outer, context, true);
-               
+                       parse_box(p, os, 0, FLAG_ITEM, outer, context, "", "", t.cs());
+
+               else if (t.cs() == "ovalbox" || t.cs() == "Ovalbox" ||
+                        t.cs() == "shadowbox" || t.cs() == "doublebox")
+                       parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), "");
+
+               else if (t.cs() == "framebox") {
+                       string special = p.getFullOpt();
+                       special += p.getOpt();
+                       parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), special);
+               }
+
                //\makebox() is part of the picture environment and different from \makebox{}
                //\makebox{} will be parsed by parse_box when bug 2956 is fixed
                else if (t.cs() == "makebox") {
@@ -2433,15 +3211,16 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        begin_inset(os, "VSpace ");
                        os << t.cs();
                        end_inset(os);
-                       skip_braces(p);
+                       skip_spaces_braces(p);
                }
 
                else if (is_known(t.cs(), known_spaces)) {
                        char const * const * where = is_known(t.cs(), known_spaces);
                        context.check_layout(os);
-                       os << "\\InsetSpace ";
+                       begin_inset(os, "space ");
                        os << '\\' << known_coded_spaces[where - known_spaces]
                           << '\n';
+                       end_inset(os);
                        // LaTeX swallows whitespace after all spaces except
                        // "\\,". We have to do that here, too, because LyX
                        // adds "{}" which would make the spaces significant.
@@ -2456,19 +3235,28 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                }
 
                else if (t.cs() == "newpage" ||
-                       t.cs() == "pagebreak" ||
-                       t.cs() == "clearpage" ||
-                       t.cs() == "cleardoublepage") {
+                        (t.cs() == "pagebreak" && !p.hasOpt()) ||
+                        t.cs() == "clearpage" ||
+                        t.cs() == "cleardoublepage") {
                        context.check_layout(os);
-                       os << "\n\\" << t.cs() << "\n";
-                       skip_braces(p); // eat {}
+                       begin_inset(os, "Newpage ");
+                       os << t.cs();
+                       end_inset(os);
+                       skip_spaces_braces(p);
                }
 
-               else if (t.cs() == "newcommand" ||
+               else if (t.cs() == "DeclareRobustCommand" ||
+                        t.cs() == "DeclareRobustCommandx" ||
+                        t.cs() == "newcommand" ||
+                        t.cs() == "newcommandx" ||
                         t.cs() == "providecommand" ||
-                        t.cs() == "renewcommand") {
-                       // these could be handled by parse_command(), but
-                       // we need to call add_known_command() here.
+                        t.cs() == "providecommandx" ||
+                        t.cs() == "renewcommand" ||
+                        t.cs() == "renewcommandx") {
+                       // DeclareRobustCommand, DeclareRobustCommandx,
+                       // providecommand and providecommandx could be handled
+                       // by parse_command(), but we need to call
+                       // add_known_command() here.
                        string name = t.asInput();
                        if (p.next_token().asInput() == "*") {
                                // Starred form. Eat '*'
@@ -2476,29 +3264,70 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                name += '*';
                        }
                        string const command = p.verbatim_item();
-                       string const opt1 = p.getOpt();
+                       string const opt1 = p.getFullOpt();
                        string const opt2 = p.getFullOpt();
                        add_known_command(command, opt1, !opt2.empty());
                        string const ert = name + '{' + command + '}' +
                                           opt1 + opt2 +
                                           '{' + p.verbatim_item() + '}';
 
-                       context.check_layout(os);
-                       begin_inset(os, "FormulaMacro");
-                       os << "\n" << ert;
-                       end_inset(os);
+                       if (t.cs() == "DeclareRobustCommand" ||
+                           t.cs() == "DeclareRobustCommandx" ||
+                           t.cs() == "providecommand" ||
+                           t.cs() == "providecommandx" ||
+                           name[name.length()-1] == '*')
+                               handle_ert(os, ert, context);
+                       else {
+                               context.check_layout(os);
+                               begin_inset(os, "FormulaMacro");
+                               os << "\n" << ert;
+                               end_inset(os);
+                       }
                }
-               
-               else if (t.cs() == "vspace") {
+
+               else if (t.cs() == "let" && p.next_token().asInput() != "*") {
+                       // let could be handled by parse_command(),
+                       // but we need to call add_known_command() here.
+                       string ert = t.asInput();
+                       string name;
+                       p.skip_spaces();
+                       if (p.next_token().cat() == catBegin) {
+                               name = p.verbatim_item();
+                               ert += '{' + name + '}';
+                       } else {
+                               name = p.verbatim_item();
+                               ert += name;
+                       }
+                       string command;
+                       p.skip_spaces();
+                       if (p.next_token().cat() == catBegin) {
+                               command = p.verbatim_item();
+                               ert += '{' + command + '}';
+                       } else {
+                               command = p.verbatim_item();
+                               ert += command;
+                       }
+                       // If command is known, make name known too, to parse
+                       // its arguments correctly. For this reason we also
+                       // have commands in syntax.default that are hardcoded.
+                       CommandMap::iterator it = known_commands.find(command);
+                       if (it != known_commands.end())
+                               known_commands[t.asInput()] = it->second;
+                       handle_ert(os, ert, context);
+               }
+
+               else if (t.cs() == "hspace" || t.cs() == "vspace") {
                        bool starred = false;
                        if (p.next_token().asInput() == "*") {
                                p.get_token();
                                starred = true;
                        }
+                       string name = t.asInput();
                        string const length = p.verbatim_item();
                        string unit;
                        string valstring;
                        bool valid = splitLatexLength(length, valstring, unit);
+                       bool known_hspace = false;
                        bool known_vspace = false;
                        bool known_unit = false;
                        double value;
@@ -2506,21 +3335,31 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                istringstream iss(valstring);
                                iss >> value;
                                if (value == 1.0) {
-                                       if (unit == "\\smallskipamount") {
-                                               unit = "smallskip";
-                                               known_vspace = true;
-                                       } else if (unit == "\\medskipamount") {
-                                               unit = "medskip";
-                                               known_vspace = true;
-                                       } else if (unit == "\\bigskipamount") {
-                                               unit = "bigskip";
-                                               known_vspace = true;
-                                       } else if (unit == "\\fill") {
-                                               unit = "vfill";
-                                               known_vspace = true;
+                                       if (t.cs()[0] == 'h') {
+                                               if (unit == "\\fill") {
+                                                       if (!starred) {
+                                                               unit = "";
+                                                               name = "\\hfill";
+                                                       }
+                                                       known_hspace = true;
+                                               }
+                                       } else {
+                                               if (unit == "\\smallskipamount") {
+                                                       unit = "smallskip";
+                                                       known_vspace = true;
+                                               } else if (unit == "\\medskipamount") {
+                                                       unit = "medskip";
+                                                       known_vspace = true;
+                                               } else if (unit == "\\bigskipamount") {
+                                                       unit = "bigskip";
+                                                       known_vspace = true;
+                                               } else if (unit == "\\fill") {
+                                                       unit = "vfill";
+                                                       known_vspace = true;
+                                               }
                                        }
                                }
-                               if (!known_vspace) {
+                               if (!known_hspace && !known_vspace) {
                                        switch (unitFromString(unit)) {
                                        case Length::SP:
                                        case Length::PT:
@@ -2542,8 +3381,23 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                }
                        }
 
-                       if (known_unit || known_vspace) {
-                               // Literal length or known variable
+                       if (t.cs()[0] == 'h' && (known_unit || known_hspace)) {
+                               // Literal horizontal length or known variable
+                               context.check_layout(os);
+                               begin_inset(os, "space ");
+                               os << name;
+                               if (starred)
+                                       os << '*';
+                               os << '{';
+                               if (known_hspace)
+                                       os << unit;
+                               os << "}";
+                               if (known_unit && !known_hspace)
+                                       os << "\n\\length "
+                                          << translate_len(length);
+                               end_inset(os);
+                       } else if (known_unit || known_vspace) {
+                               // Literal vertical length or known variable
                                context.check_layout(os);
                                begin_inset(os, "VSpace ");
                                if (known_unit)
@@ -2553,8 +3407,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        os << '*';
                                end_inset(os);
                        } else {
-                               // LyX can't handle other length variables in Inset VSpace
-                               string name = t.asInput();
+                               // LyX can't handle other length variables in Inset VSpace/space
                                if (starred)
                                        name += '*';
                                if (valid) {
@@ -2569,10 +3422,24 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                }
 
+               // The single '=' is meant here.
+               else if ((newinsetlayout = findInsetLayout(context.textclass, t.cs(), true))) {
+                       p.skip_spaces();
+                       context.check_layout(os);
+                       begin_inset(os, "Flex ");
+                       os << to_utf8(newinsetlayout->name()) << '\n'
+                          << "status collapsed\n";
+                       parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
+                       end_inset(os);
+               }
+
                else {
                        // try to see whether the string is in unicodesymbols
+                       // Only use text mode commands, since we are in text mode here,
+                       // and math commands may be invalid (bug 6797)
                        docstring rem;
-                       docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()), rem);
+                       docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
+                                                                rem, Encodings::TEXT_CMD);
                        if (!s.empty()) {
                                if (!rem.empty())
                                        cerr << "When parsing " << t.cs() 
@@ -2580,8 +3447,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                             << "+" << to_utf8(rem) << endl;
                                context.check_layout(os);
                                os << to_utf8(s);
-                               p.skip_spaces();
-                               skip_braces(p); // eat {}
+                               skip_spaces_braces(p);
                        }
                        //cerr << "#: " << t << " mode: " << mode << endl;
                        // heuristic: read up to next non-nested space