]> git.lyx.org Git - lyx.git/blobdiff - src/tex2lyx/text.cpp
fix msvc linker error: for the linker struct and class isn't the same here
[lyx.git] / src / tex2lyx / text.cpp
index be604629b399e4e47c445552a89fbc725acc242d..0c49bba9dafe85fc0e362e5a3736a6d1a535bc25 100644 (file)
@@ -41,10 +41,15 @@ namespace lyx {
 
 
 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
-               Context const & context)
+               Context const & context, InsetLayout const * layout)
 {
+       bool const forcePlainLayout =
+               layout ? layout->forcePlainLayout() : false;
        Context newcontext(true, context.textclass);
-       newcontext.font = context.font;
+       if (forcePlainLayout)
+               newcontext.layout = &context.textclass.plainLayout();
+       else
+               newcontext.font = context.font;
        parse_text(p, os, flags, outer, newcontext);
        newcontext.check_end_layout(os);
 }
@@ -52,6 +57,17 @@ void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
 
 namespace {
 
+void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
+               Context const & context, string const & name)
+{
+       InsetLayout const * layout = 0;
+       DocumentClass::InsetLayouts::const_iterator it =
+               context.textclass.insetLayouts().find(from_ascii(name));
+       if (it != context.textclass.insetLayouts().end())
+               layout = &(it->second);
+       parse_text_in_inset(p, os, flags, outer, context, layout);
+}
+
 /// parses a paragraph snippet, useful for example for \\emph{...}
 void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
                Context & context)
@@ -393,6 +409,53 @@ bool skip_braces(Parser & p)
 }
 
 
+/// replace LaTeX commands in \p s from the unicodesymbols file with their
+/// unciode points
+docstring convert_unicodesymbols(docstring s)
+{
+       odocstringstream os;
+       for (size_t i = 0; i < s.size();) {
+               if (s[i] != '\\') {
+                       os << s[i++];
+                       continue;
+               }
+               s = s.substr(i);
+               docstring rem;
+               docstring parsed = encodings.fromLaTeXCommand(s, rem,
+                               Encodings::TEXT_CMD);
+               os << parsed;
+               s = rem;
+               if (s.empty() || s[0] != '\\')
+                       i = 0;
+               else
+                       i = 1;
+       }
+       return os.str();
+}
+
+
+/// try to convert \p s to a valid InsetCommand argument
+string convert_command_inset_arg(string s)
+{
+       if (isAscii(s))
+               // since we don't know the input encoding we can't use from_utf8
+               s = to_utf8(convert_unicodesymbols(from_ascii(s)));
+       // LyX cannot handle newlines in a latex command
+       return subst(s, "\n", " ");
+}
+
+
+void handle_backslash(ostream & os, string const & s)
+{
+       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
+               if (*it == '\\')
+                       os << "\n\\backslash\n";
+               else
+                       os << *it;
+       }
+}
+
+
 void handle_ert(ostream & os, string const & s, Context & context)
 {
        // We must have a valid layout before outputting the ERT inset.
@@ -422,12 +485,7 @@ void handle_comment(ostream & os, string const & s, Context & context)
        begin_inset(os, "ERT");
        os << "\nstatus collapsed\n";
        newcontext.check_layout(os);
-       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
-               if (*it == '\\')
-                       os << "\n\\backslash\n";
-               else
-                       os << *it;
-       }
+       handle_backslash(os, s);
        // make sure that our comment is the last thing on the line
        newcontext.new_paragraph(os);
        newcontext.check_layout(os);
@@ -436,14 +494,25 @@ void handle_comment(ostream & os, string const & s, Context & context)
 }
 
 
-Layout const * findLayout(TextClass const & textclass, string const & name)
+Layout const * findLayout(TextClass const & textclass, string const & name, bool command)
+{
+       Layout const * layout = findLayoutWithoutModule(textclass, name, command);
+       if (layout)
+               return layout;
+       if (checkModule(name, command))
+               return findLayoutWithoutModule(textclass, name, command);
+       return layout;
+}
+
+
+InsetLayout const * findInsetLayout(TextClass const & textclass, string const & name, bool command)
 {
-       DocumentClass::const_iterator lit = textclass.begin();
-       DocumentClass::const_iterator len = textclass.end();
-       for (; lit != len; ++lit)
-               if (lit->latexname() == name)
-                       return &*lit;
-       return 0;
+       InsetLayout const * insetlayout = findInsetLayoutWithoutModule(textclass, name, command);
+       if (insetlayout)
+               return insetlayout;
+       if (checkModule(name, command))
+               return findInsetLayoutWithoutModule(textclass, name, command);
+       return insetlayout;
 }
 
 
@@ -614,7 +683,13 @@ void parse_arguments(string const & command,
                        break;
                case optional:
                        // true because we must not eat whitespace
-                       ert += p.getOpt(true);
+                       // if an optional arg follows me must not strip the
+                       // brackets from this one
+                       if (i < no_arguments - 1 &&
+                           template_arguments[i+1] == optional)
+                               ert += p.getFullOpt(true);
+                       else
+                               ert += p.getOpt(true);
                        break;
                }
        }
@@ -640,22 +715,26 @@ bool parse_command(string const & command, Parser & p, ostream & os,
 
 
 /// Parses a minipage or parbox
-void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
-              Context & parent_context, bool use_parbox)
+void parse_box(Parser & p, ostream & os, unsigned outer_flags,
+               unsigned inner_flags, bool outer, Context & parent_context,
+               string const & outer_type, string const & special,
+               string const & inner_type)
 {
        string position;
        string inner_pos;
+       string hor_pos = "c";
        // We need to set the height to the LaTeX default of 1\\totalheight
        // for the case when no height argument is given
        string height_value = "1";
        string height_unit = "in";
        string height_special = "totalheight";
        string latex_height;
-       if (p.hasOpt()) {
+       if (!inner_type.empty() && p.hasOpt()) {
                position = p.getArg('[', ']');
                if (position != "t" && position != "c" && position != "b") {
+                       cerr << "invalid position " << position << " for "
+                            << inner_type << endl;
                        position = "c";
-                       cerr << "invalid position for minipage/parbox" << endl;
                }
                if (p.hasOpt()) {
                        latex_height = p.getArg('[', ']');
@@ -665,40 +744,100 @@ void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
                                inner_pos = p.getArg('[', ']');
                                if (inner_pos != "c" && inner_pos != "t" &&
                                    inner_pos != "b" && inner_pos != "s") {
+                                       cerr << "invalid inner_pos "
+                                            << inner_pos << " for "
+                                            << inner_type << endl;
                                        inner_pos = position;
-                                       cerr << "invalid inner_pos for minipage/parbox"
-                                            << endl;
                                }
                        }
                }
        }
        string width_value;
        string width_unit;
-       string const latex_width = p.verbatim_item();
+       string latex_width;
+       if (inner_type.empty()) {
+               if (special.empty())
+                       latex_width = "\\columnwidth";
+               else {
+                       Parser p2(special);
+                       latex_width = p2.getArg('[', ']');
+                       string const opt = p2.getArg('[', ']');
+                       if (!opt.empty()) {
+                               hor_pos = opt;
+                               if (hor_pos != "l" && hor_pos != "c" &&
+                                   hor_pos != "r") {
+                                       cerr << "invalid hor_pos " << hor_pos
+                                            << " for " << outer_type << endl;
+                                       hor_pos = "c";
+                               }
+                       }
+               }
+       } else
+               latex_width = p.verbatim_item();
        translate_len(latex_width, width_value, width_unit);
-       if (contains(width_unit, '\\') || contains(height_unit, '\\')) {
-               // LyX can't handle length variables
-               ostringstream ss;
-               if (use_parbox)
-                       ss << "\\parbox";
+       // LyX can't handle length variables
+       bool use_ert = contains(width_unit, '\\') || contains(height_unit, '\\');
+       if (!use_ert && !outer_type.empty() && !inner_type.empty()) {
+               // Look whether there is some content after the end of the
+               // inner box, but before the end of the outer box.
+               // If yes, we need to output ERT.
+               p.pushPosition();
+               if (inner_flags & FLAG_END)
+                       p.verbatimEnvironment(inner_type);
                else
-                       ss << "\\begin{minipage}";
-               if (!position.empty())
-                       ss << '[' << position << ']';
-               if (!latex_height.empty())
-                       ss << '[' << latex_height << ']';
-               if (!inner_pos.empty())
-                       ss << '[' << inner_pos << ']';
-               ss << "{" << latex_width << "}";
-               if (use_parbox)
-                       ss << '{';
+                       p.verbatim_item();
+               p.skip_spaces(true);
+               if ((outer_type == "framed" && p.next_token().asInput() != "\\end") ||
+                   (outer_type != "framed" && p.next_token().cat() != catEnd)) {
+                       // something is between the end of the inner box and
+                       // the end of the outer box, so we need to use ERT.
+                       use_ert = true;
+               }
+               p.popPosition();
+       }
+       if (use_ert) {
+               ostringstream ss;
+               if (!outer_type.empty()) {
+                       if (outer_flags & FLAG_END)
+                               ss << "\\begin{" << outer_type << '}';
+                       else {
+                               ss << '\\' << outer_type << '{';
+                               if (!special.empty())
+                                       ss << special;
+                       }
+               }
+               if (!inner_type.empty()) {
+                       if (inner_flags & FLAG_END)
+                               ss << "\\begin{" << inner_type << '}';
+                       else
+                               ss << '\\' << inner_type;
+                       if (!position.empty())
+                               ss << '[' << position << ']';
+                       if (!latex_height.empty())
+                               ss << '[' << latex_height << ']';
+                       if (!inner_pos.empty())
+                               ss << '[' << inner_pos << ']';
+                       ss << '{' << latex_width << '}';
+                       if (!(inner_flags & FLAG_END))
+                               ss << '{';
+               }
                handle_ert(os, ss.str(), parent_context);
-               parent_context.new_paragraph(os);
-               parse_text_in_inset(p, os, flags, outer, parent_context);
-               if (use_parbox)
-                       handle_ert(os, "}", parent_context);
-               else
-                       handle_ert(os, "\\end{minipage}", parent_context);
+               if (!inner_type.empty()) {
+                       parse_text(p, os, inner_flags, outer, parent_context);
+                       if (inner_flags & FLAG_END)
+                               handle_ert(os, "\\end{" + inner_type + '}',
+                                          parent_context);
+                       else
+                               handle_ert(os, "}", parent_context);
+               }
+               if (!outer_type.empty()) {
+                       parse_text(p, os, outer_flags, outer, parent_context);
+                       if (outer_flags & FLAG_END)
+                               handle_ert(os, "\\end{" + outer_type + '}',
+                                          parent_context);
+                       else
+                               handle_ert(os, "}", parent_context);
+               }
        } else {
                // LyX does not like empty positions, so we have
                // to set them to the LaTeX default values here.
@@ -707,18 +846,54 @@ void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
                if (inner_pos.empty())
                        inner_pos = position;
                parent_context.check_layout(os);
-               begin_inset(os, "Box Frameless\n");
+               begin_inset(os, "Box ");
+               if (outer_type == "framed")
+                       os << "Framed\n";
+               else if (outer_type == "framebox")
+                       os << "Boxed\n";
+               else if (outer_type == "shadowbox")
+                       os << "Shadowbox\n";
+               else if (outer_type == "shaded")
+                       os << "Shaded\n";
+               else if (outer_type == "doublebox")
+                       os << "Doublebox\n";
+               else if (outer_type.empty())
+                       os << "Frameless\n";
+               else
+                       os << outer_type << '\n';
                os << "position \"" << position << "\"\n";
-               os << "hor_pos \"c\"\n";
-               os << "has_inner_box 1\n";
+               os << "hor_pos \"" << hor_pos << "\"\n";
+               os << "has_inner_box " << !inner_type.empty() << "\n";
                os << "inner_pos \"" << inner_pos << "\"\n";
-               os << "use_parbox " << use_parbox << "\n";
+               os << "use_parbox " << (inner_type == "parbox") << '\n';
                os << "width \"" << width_value << width_unit << "\"\n";
                os << "special \"none\"\n";
                os << "height \"" << height_value << height_unit << "\"\n";
                os << "height_special \"" << height_special << "\"\n";
                os << "status open\n\n";
-               parse_text_in_inset(p, os, flags, outer, parent_context);
+               Context context(true, parent_context.textclass);
+               context.font = parent_context.font;
+
+               // If we have no inner box the contens will be read with the outer box
+               if (!inner_type.empty())
+                       parse_text(p, os, inner_flags, outer, context);
+
+               // Ensure that the end of the outer box is parsed correctly:
+               // The opening brace has been eaten by parse_outer_box()
+               if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
+                       outer_flags &= ~FLAG_ITEM;
+                       outer_flags |= FLAG_BRACE_LAST;
+               }
+
+               // Find end of outer box, output contents if inner_type is
+               // empty and output possible comments
+               if (!outer_type.empty()) {
+                       // This does not output anything but comments if
+                       // inner_type is not empty (see use_ert)
+                       parse_text(p, os, outer_flags, outer, context);
+               }
+
+               context.check_end_layout(os);
                end_inset(os);
 #ifdef PRESERVE_LAYOUT
                // LyX puts a % after the end of the minipage
@@ -744,6 +919,81 @@ void parse_box(Parser & p, ostream & os, unsigned flags, bool outer,
 }
 
 
+void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer,
+                     Context & parent_context, string const & outer_type,
+                     string const & special)
+{
+       eat_whitespace(p, os, parent_context, false);
+       if (flags & FLAG_ITEM) {
+               // Eat '{'
+               if (p.next_token().cat() == catBegin)
+                       p.get_token();
+               else
+                       cerr << "Warning: Ignoring missing '{' after \\"
+                            << outer_type << '.' << endl;
+               eat_whitespace(p, os, parent_context, false);
+       }
+       string inner;
+       unsigned int inner_flags = 0;
+       if (outer_type == "shaded") {
+               // These boxes never have an inner box
+               ;
+       } else if (p.next_token().asInput() == "\\parbox") {
+               inner = p.get_token().cs();
+               inner_flags = FLAG_ITEM;
+       } else if (p.next_token().asInput() == "\\begin") {
+               // Is this a minipage?
+               p.pushPosition();
+               p.get_token();
+               inner = p.getArg('{', '}');
+               p.popPosition();
+               if (inner == "minipage") {
+                       p.get_token();
+                       p.getArg('{', '}');
+                       eat_whitespace(p, os, parent_context, false);
+                       inner_flags = FLAG_END;
+               } else
+                       inner = "";
+       }
+       if (inner_flags == FLAG_END) {
+               active_environments.push_back(inner);
+               parse_box(p, os, flags, FLAG_END, outer, parent_context,
+                         outer_type, special, inner);
+               active_environments.pop_back();
+       } else {
+               parse_box(p, os, flags, inner_flags, outer, parent_context,
+                         outer_type, special, inner);
+       }
+}
+
+
+void parse_listings(Parser & p, ostream & os, Context & parent_context)
+{
+       parent_context.check_layout(os);
+       begin_inset(os, "listings\n");
+       os << "inline false\n"
+          << "status collapsed\n";
+       Context context(true, parent_context.textclass);
+       context.layout = &parent_context.textclass.plainLayout();
+       context.check_layout(os);
+       string const s = p.verbatimEnvironment("lstlisting");
+       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
+               if (*it == '\\')
+                       os << "\n\\backslash\n";
+               else if (*it == '\n') {
+                       // avoid adding an empty paragraph at the end
+                       if (it + 1 != et) {
+                               context.new_paragraph(os);
+                               context.check_layout(os);
+                       }
+               } else
+                       os << *it;
+       }
+       context.check_end_layout(os);
+       end_inset(os);
+}
+
+
 /// parse an unknown environment
 void parse_unknown_environment(Parser & p, string const & name, ostream & os,
                               unsigned flags, bool outer,
@@ -775,6 +1025,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        string & last_env, Context & parent_context)
 {
        Layout const * newlayout;
+       InsetLayout const * newinsetlayout = 0;
        string const name = p.getArg('{', '}');
        const bool is_starred = suffixIs(name, '*');
        string const unstarred_name = rtrim(name, "*");
@@ -817,7 +1068,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 
        else if (name == "minipage") {
                eat_whitespace(p, os, parent_context, false);
-               parse_box(p, os, FLAG_END, outer, parent_context, false);
+               parse_box(p, os, 0, FLAG_END, outer, parent_context, "", "", name);
                p.skip_spaces();
        }
 
@@ -844,23 +1095,18 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 
        else if (name == "framed" || name == "shaded") {
                eat_whitespace(p, os, parent_context, false);
-               parent_context.check_layout(os);
-               if (name == "framed")
-                       begin_inset(os, "Box Framed\n");
+               parse_outer_box(p, os, FLAG_END, outer, parent_context, name, "");
+               p.skip_spaces();
+       }
+
+       else if (name == "lstlisting") {
+               eat_whitespace(p, os, parent_context, false);
+               // FIXME handle listings with parameters
+               if (p.hasOpt())
+                       parse_unknown_environment(p, name, os, FLAG_END,
+                                                 outer, parent_context);
                else
-                       begin_inset(os, "Box Shaded\n");
-               os << "position \"t\"\n"
-                     "hor_pos \"c\"\n"
-                     "has_inner_box 0\n"
-                     "inner_pos \"t\"\n"
-                     "use_parbox 0\n"
-                     "width \"100col%\"\n"
-                     "special \"none\"\n"
-                     "height \"1in\"\n"
-                     "height_special \"totalheight\"\n"
-                     "status open\n";
-               parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
-               end_inset(os);
+                       parse_listings(p, os, parent_context);
                p.skip_spaces();
        }
 
@@ -908,8 +1154,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
        }
 
        // The single '=' is meant here.
-       else if ((newlayout = findLayout(parent_context.textclass, name)) &&
-                 newlayout->isEnvironment()) {
+       else if ((newlayout = findLayout(parent_context.textclass, name, false))) {
                eat_whitespace(p, os, parent_context, false);
                Context context(true, parent_context.textclass, newlayout,
                                parent_context.layout, parent_context.font);
@@ -968,6 +1213,17 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                p.skip_spaces();
        }
 
+       // The single '=' is meant here.
+       else if ((newinsetlayout = findInsetLayout(parent_context.textclass, name, false))) {
+               eat_whitespace(p, os, parent_context, false);
+               parent_context.check_layout(os);
+               begin_inset(os, "Flex ");
+               os << to_utf8(newinsetlayout->name()) << '\n'
+                  << "status collapsed\n";
+               parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
+               end_inset(os);
+       }
+
        else if (name == "appendix") {
                // This is no good latex style, but it works and is used in some documents...
                eat_whitespace(p, os, parent_context, false);
@@ -1334,6 +1590,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                Context & context)
 {
        Layout const * newlayout = 0;
+       InsetLayout const * newinsetlayout = 0;
        // Store the latest bibliographystyle and nocite{*} option
        // (needed for bibtex inset)
        string btprint;
@@ -1702,10 +1959,19 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "bibitem") {
                        context.set_item();
                        context.check_layout(os);
-                       begin_command_inset(os, "bibitem", "bibitem");
-                       os << "label \"" << p.getOptContent() << "\"\n";
-                       os << "key \"" << p.verbatim_item() << "\"\n";
-                       end_inset(os);
+                       string label = convert_command_inset_arg(p.getArg('[', ']'));
+                       string key = convert_command_inset_arg(p.verbatim_item());
+                       if (contains(label, '\\') || contains(key, '\\')) {
+                               // LyX can't handle LaTeX commands in labels or keys
+                               handle_ert(os, t.asInput() + '[' + label +
+                                              "]{" + p.verbatim_item() + '}',
+                                          context);
+                       } else {
+                               begin_command_inset(os, "bibitem", "bibitem");
+                               os << "label \"" << label << "\"\n"
+                                     "key \"" << key << "\"\n";
+                               end_inset(os);
+                       }
                }
 
                else if (is_macro(p))
@@ -1743,8 +2009,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                // Must attempt to parse "Section*" before "Section".
                else if ((p.next_token().asInput() == "*") &&
                         context.new_layout_allowed &&
-                        (newlayout = findLayout(context.textclass, t.cs() + '*')) &&
-                        newlayout->isCommand()) {
+                        (newlayout = findLayout(context.textclass, t.cs() + '*', true))) {
                        // write the layout
                        p.get_token();
                        output_command_layout(os, p, outer, context, newlayout);
@@ -1753,8 +2018,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                // Section headings and the like
                else if (context.new_layout_allowed &&
-                        (newlayout = findLayout(context.textclass, t.cs())) &&
-                        newlayout->isCommand()) {
+                        (newlayout = findLayout(context.textclass, t.cs(), true))) {
                        // write the layout
                        output_command_layout(os, p, outer, context, newlayout);
                        p.skip_spaces();
@@ -2120,22 +2384,34 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        os << "\\lyxline";
                }
 
-               else if (is_known(t.cs(), known_phrases)) {
+               else if (is_known(t.cs(), known_phrases) ||
+                        (t.cs() == "protect" &&
+                         p.next_token().cat() == catEscape &&
+                         is_known(p.next_token().cs(), known_phrases))) {
+                       // LyX sometimes puts a \protect in front, so we have to ignore it
                        // FIXME: This needs to be changed when bug 4752 is fixed.
-                       char const * const * where = is_known(t.cs(), known_phrases);
+                       char const * const * where = is_known(
+                               t.cs() == "protect" ? p.get_token().cs() : t.cs(),
+                               known_phrases);
                        context.check_layout(os);
                        os << known_coded_phrases[where - known_phrases];
                        skip_spaces_braces(p);
                }
 
                else if (is_known(t.cs(), known_ref_commands)) {
-                       context.check_layout(os);
-                       begin_command_inset(os, "ref", t.cs());
-                       // LyX cannot handle newlines in a latex command
-                       // FIXME: Move the substitution into parser::getOpt()?
-                       os << subst(p.getOpt(), "\n", " ");
-                       os << "reference " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
-                       end_inset(os);
+                       string const opt = p.getOpt();
+                       if (opt.empty()) {
+                               context.check_layout(os);
+                               begin_command_inset(os, "ref", t.cs());
+                               os << "reference \""
+                                  << convert_command_inset_arg(p.verbatim_item())
+                                  << "\"\n";
+                               end_inset(os);
+                       } else {
+                               // LyX does not support optional arguments of ref commands
+                               handle_ert(os, t.asInput() + '[' + opt + "]{" +
+                                              p.verbatim_item() + "}", context);
+                       }
                }
 
                else if (use_natbib &&
@@ -2181,19 +2457,19 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        if (!after.empty()) {
                                after.erase(0, 1);
                                after.erase(after.length() - 1, 1);
-                               // LyX cannot handle newlines in the parameter
-                               after = subst(after, "\n", " ");
+                               after = convert_command_inset_arg(after);
                        }
                        if (!before.empty()) {
                                before.erase(0, 1);
                                before.erase(before.length() - 1, 1);
-                               // LyX cannot handle newlines in the parameter
-                               before = subst(before, "\n", " ");
+                               before = convert_command_inset_arg(before);
                        }
                        begin_command_inset(os, "citation", command);
                        os << "after " << '"' << after << '"' << "\n";
                        os << "before " << '"' << before << '"' << "\n";
-                       os << "key " << '"' << p.verbatim_item() << '"' << "\n";
+                       os << "key \""
+                          << convert_command_inset_arg(p.verbatim_item())
+                          << "\"\n";
                        end_inset(os);
                }
 
@@ -2249,9 +2525,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "cite"
                        || t.cs() == "nocite") {
                        context.check_layout(os);
-                       // LyX cannot handle newlines in a latex command
-                       string after = subst(p.getOptContent(), "\n", " ");
-                       string key = subst(p.verbatim_item(), "\n", " ");
+                       string after = convert_command_inset_arg(p.getArg('[', ']'));
+                       string key = convert_command_inset_arg(p.verbatim_item());
                        // store the case that it is "\nocite{*}" to use it later for
                        // the BibTeX inset
                        if (key != "*") {
@@ -2267,27 +2542,30 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        context.check_layout(os);
                        begin_inset(os, "Index\n");
                        os << "status collapsed\n";
-                       parse_text_in_inset(p, os, FLAG_ITEM, false, context);
+                       parse_text_in_inset(p, os, FLAG_ITEM, false, context, "Index");
                        end_inset(os);
                }
 
                else if (t.cs() == "nomenclature") {
                        context.check_layout(os);
                        begin_command_inset(os, "nomenclature", "nomenclature");
-                       // LyX cannot handle newlines in a latex command
-                       string prefix = subst(p.getOptContent(), "\n", " ");
+                       string prefix = convert_command_inset_arg(p.getArg('[', ']'));
                        if (!prefix.empty())
                                os << "prefix " << '"' << prefix << '"' << "\n";
-                       os << "symbol " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
-                       os << "description " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       os << "symbol " << '"'
+                          << convert_command_inset_arg(p.verbatim_item());
+                       os << "\"\ndescription \""
+                          << convert_command_inset_arg(p.verbatim_item())
+                          << "\"\n";
                        end_inset(os);
                }
                
                else if (t.cs() == "label") {
                        context.check_layout(os);
                        begin_command_inset(os, "label", "label");
-                       // LyX cannot handle newlines in a latex command
-                       os << "name " << '"' << subst(p.verbatim_item(), "\n", " ") << '"' << "\n";
+                       os << "name \""
+                          << convert_command_inset_arg(p.verbatim_item())
+                          << "\"\n";
                        end_inset(os);
                }
 
@@ -2305,14 +2583,6 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        skip_spaces_braces(p);
                }
 
-               else if (t.cs() == "url") {
-                       context.check_layout(os);
-                       begin_inset(os, "Flex URL\n");
-                       os << "status collapsed\n";
-                       parse_text_in_inset(p, os, FLAG_ITEM, false, context);
-                       end_inset(os);
-               }
-
                else if (LYX_FORMAT >= 408 &&
                         (t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
                        context.check_layout(os);
@@ -2427,21 +2697,15 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // save the language for the case that a
                        // \foreignlanguage is used 
 
-                       // FIXME: \lang needs a LyX name, but we set a LaTeX name
-                       context.font.language = subst(p.verbatim_item(), "\n", " ");
-                       os << "\\lang " << context.font.language << "\n";
+                       context.font.language = babel2lyx(p.verbatim_item());
+                       os << "\n\\lang " << context.font.language << "\n";
                }
 
                else if (t.cs() == "foreignlanguage") {
-                       context.check_layout(os);
-                       // FIXME: \lang needs a LyX name, but we set a LaTeX name
-                       os << "\n\\lang " << subst(p.verbatim_item(), "\n", " ") << "\n";
-                       os << subst(p.verbatim_item(), "\n", " ");
-                       // FIXME: the second argument of selectlanguage
-                       // has to be parsed (like for \textsf, for
-                       // example). 
-                       // set back to last selectlanguage
-                       os << "\n\\lang " << context.font.language << "\n";
+                       string const lang = babel2lyx(p.verbatim_item());
+                       parse_text_attributes(p, os, FLAG_ITEM, outer,
+                                             context, "\\lang",
+                                             context.font.language, lang);
                }
 
                else if (t.cs() == "inputencoding") {
@@ -2752,8 +3016,18 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                }
 
                else if (t.cs() == "parbox")
-                       parse_box(p, os, FLAG_ITEM, outer, context, true);
-               
+                       parse_box(p, os, 0, FLAG_ITEM, outer, context, "", "", t.cs());
+
+               else if (t.cs() == "ovalbox" || t.cs() == "Ovalbox" ||
+                        t.cs() == "shadowbox" || t.cs() == "doublebox")
+                       parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), "");
+
+               else if (t.cs() == "framebox") {
+                       string special = p.getFullOpt();
+                       special += p.getOpt();
+                       parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), special);
+               }
+
                //\makebox() is part of the picture environment and different from \makebox{}
                //\makebox{} will be parsed by parse_box when bug 2956 is fixed
                else if (t.cs() == "makebox") {
@@ -2809,11 +3083,18 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        skip_spaces_braces(p);
                }
 
-               else if (t.cs() == "newcommand" ||
+               else if (t.cs() == "DeclareRobustCommand" ||
+                        t.cs() == "DeclareRobustCommandx" ||
+                        t.cs() == "newcommand" ||
+                        t.cs() == "newcommandx" ||
                         t.cs() == "providecommand" ||
-                        t.cs() == "renewcommand") {
-                       // providecommand could be handled by parse_command(),
-                       // but we need to call add_known_command() here.
+                        t.cs() == "providecommandx" ||
+                        t.cs() == "renewcommand" ||
+                        t.cs() == "renewcommandx") {
+                       // DeclareRobustCommand, DeclareRobustCommandx,
+                       // providecommand and providecommandx could be handled
+                       // by parse_command(), but we need to call
+                       // add_known_command() here.
                        string name = t.asInput();
                        if (p.next_token().asInput() == "*") {
                                // Starred form. Eat '*'
@@ -2821,14 +3102,17 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                name += '*';
                        }
                        string const command = p.verbatim_item();
-                       string const opt1 = p.getOpt();
+                       string const opt1 = p.getFullOpt();
                        string const opt2 = p.getFullOpt();
                        add_known_command(command, opt1, !opt2.empty());
                        string const ert = name + '{' + command + '}' +
                                           opt1 + opt2 +
                                           '{' + p.verbatim_item() + '}';
 
-                       if (t.cs() == "providecommand" ||
+                       if (t.cs() == "DeclareRobustCommand" ||
+                           t.cs() == "DeclareRobustCommandx" ||
+                           t.cs() == "providecommand" ||
+                           t.cs() == "providecommandx" ||
                            name[name.length()-1] == '*')
                                handle_ert(os, ert, context);
                        else {
@@ -2976,6 +3260,17 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                }
 
+               // The single '=' is meant here.
+               else if ((newinsetlayout = findInsetLayout(context.textclass, t.cs(), true))) {
+                       p.skip_spaces();
+                       context.check_layout(os);
+                       begin_inset(os, "Flex ");
+                       os << to_utf8(newinsetlayout->name()) << '\n'
+                          << "status collapsed\n";
+                       parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
+                       end_inset(os);
+               }
+
                else {
                        // try to see whether the string is in unicodesymbols
                        // Only use text mode commands, since we are in text mode here,