]> git.lyx.org Git - features.git/blobdiff - src/tex2lyx/text.cpp
Merge branch 'master' of git.lyx.org:lyx
[features.git] / src / tex2lyx / text.cpp
index 52047536b34255a3c21174a4cf2caaa34777310e..46cc347db75d8429a47028fedad1ed4e22da659a 100644 (file)
@@ -117,13 +117,13 @@ char const * const known_ref_commands[] = { "ref", "pageref", "vref",
 char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref",
  "vpageref", "formatted", "eqref", 0 };
 
-char const * const known_refstyle_commands[] = { "chapref", "corref", "eqref",
- "enuref", "fnref", "lemref", "parref", "partref", "propref", "secref", "Staref",
- "subref", "thmref", 0 };
+char const * const known_refstyle_commands[] = { "algref", "chapref", "corref",
+ "eqref", "enuref", "figref", "fnref", "lemref", "parref", "partref", "propref",
+ "secref", "subref", "tabref", "thmref", 0 };
 
-char const * const known_refstyle_prefixes[] = { "chap", "cor", "eq",
- "enu", "fn", "lem", "par", "part", "prop", "sec", "Sta",
- "sub", "thm", 0 };
+char const * const known_refstyle_prefixes[] = { "alg", "chap", "cor",
+ "eq", "enu", "fig", "fn", "lem", "par", "part", "prop",
+ "sec", "sub", "tab", "thm", 0 };
 
 
 /**
@@ -285,6 +285,18 @@ char const * const known_phrases[] = {"LyX", "TeX", "LaTeXe", "LaTeX", 0};
 char const * const known_coded_phrases[] = {"LyX", "TeX", "LaTeX2e", "LaTeX", 0};
 int const known_phrase_lengths[] = {3, 5, 7, 0};
 
+/// known TIPA combining diacritical marks
+char const * const known_tipa_marks[] = {"textsubwedge", "textsubumlaut",
+"textsubtilde", "textseagull", "textsubbridge", "textinvsubbridge",
+"textsubsquare", "textsubrhalfring", "textsublhalfring", "textsubplus",
+"textovercross", "textsubarch", "textsuperimposetilde", "textraising",
+"textlowering", "textadvancing", "textretracting", "textdoublegrave",
+"texthighrise", "textlowrise", "textrisefall", "textsyllabic",
+"textsubring", 0};
+
+/// TIPA tones that need special handling
+char const * const known_tones[] = {"15", "51", "45", "12", "454", 0};
+
 // string to store the float type to be able to determine the type of subfloats
 string float_type = "";
 
@@ -516,57 +528,33 @@ string convert_command_inset_arg(string s)
 }
 
 
-void handle_backslash(ostream & os, string const & s)
-{
-       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
-               if (*it == '\\')
-                       os << "\n\\backslash\n";
-               else
-                       os << *it;
-       }
-}
-
-
-void handle_ert(ostream & os, string const & s, Context & context)
+void output_ert(ostream & os, string const & s, Context & context)
 {
-       // We must have a valid layout before outputting the ERT inset.
        context.check_layout(os);
-       Context newcontext(true, context.textclass);
-       InsetLayout const & layout = context.textclass.insetLayout(from_ascii("ERT"));
-       if (layout.forcePlainLayout())
-               newcontext.layout = &context.textclass.plainLayout();
-       begin_inset(os, "ERT");
-       os << "\nstatus collapsed\n";
-       newcontext.check_layout(os);
        for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
                if (*it == '\\')
                        os << "\n\\backslash\n";
                else if (*it == '\n') {
-                       newcontext.new_paragraph(os);
-                       newcontext.check_layout(os);
+                       context.new_paragraph(os);
+                       context.check_layout(os);
                } else
                        os << *it;
        }
-       newcontext.check_end_layout(os);
-       end_inset(os);
+       context.check_end_layout(os);
 }
 
 
-void handle_comment(ostream & os, string const & s, Context & context)
+void output_ert_inset(ostream & os, string const & s, Context & context)
 {
-       // TODO: Handle this better
+       // We must have a valid layout before outputting the ERT inset.
+       context.check_layout(os);
        Context newcontext(true, context.textclass);
        InsetLayout const & layout = context.textclass.insetLayout(from_ascii("ERT"));
        if (layout.forcePlainLayout())
                newcontext.layout = &context.textclass.plainLayout();
        begin_inset(os, "ERT");
        os << "\nstatus collapsed\n";
-       newcontext.check_layout(os);
-       handle_backslash(os, s);
-       // make sure that our comment is the last thing on the line
-       newcontext.new_paragraph(os);
-       newcontext.check_layout(os);
-       newcontext.check_end_layout(os);
+       output_ert(os, s, newcontext);
        end_inset(os);
 }
 
@@ -754,7 +742,7 @@ void parse_arguments(string const & command,
                case required:
                case req_group:
                        // This argument contains regular LaTeX
-                       handle_ert(os, ert + '{', context);
+                       output_ert_inset(os, ert + '{', context);
                        eat_whitespace(p, os, context, false);
                        if (template_arguments[i] == required)
                                parse_text(p, os, FLAG_ITEM, outer, context);
@@ -789,7 +777,7 @@ void parse_arguments(string const & command,
                        break;
                }
        }
-       handle_ert(os, ert, context);
+       output_ert_inset(os, ert, context);
 }
 
 
@@ -924,7 +912,7 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
                // If yes, we need to output ERT.
                p.pushPosition();
                if (inner_flags & FLAG_END)
-                       p.verbatimEnvironment(inner_type);
+                       p.ertEnvironment(inner_type);
                else
                        p.verbatim_item();
                p.skip_spaces(true);
@@ -977,14 +965,14 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
                }
                if (inner_type == "shaded")
                        ss << "\\begin{shaded}";
-               handle_ert(os, ss.str(), parent_context);
+               output_ert_inset(os, ss.str(), parent_context);
                if (!inner_type.empty()) {
                        parse_text(p, os, inner_flags, outer, parent_context);
                        if (inner_flags & FLAG_END)
-                               handle_ert(os, "\\end{" + inner_type + '}',
+                               output_ert_inset(os, "\\end{" + inner_type + '}',
                                           parent_context);
                        else
-                               handle_ert(os, "}", parent_context);
+                               output_ert_inset(os, "}", parent_context);
                }
                if (!outer_type.empty()) {
                        // If we already read the inner box we have to pop
@@ -1000,13 +988,13 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
                        }
                        parse_text(p, os, outer_flags, outer, parent_context);
                        if (outer_flags & FLAG_END)
-                               handle_ert(os, "\\end{" + outer_type + '}',
+                               output_ert_inset(os, "\\end{" + outer_type + '}',
                                           parent_context);
                        else if (inner_type.empty() && outer_type == "framebox")
                                // in this case it is already closed later
                                ;
                        else
-                               handle_ert(os, "}", parent_context);
+                               output_ert_inset(os, "}", parent_context);
                }
        } else {
                // LyX does not like empty positions, so we have
@@ -1089,13 +1077,13 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
                // LyX puts a % after the end of the minipage
                if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) {
                        // new paragraph
-                       //handle_comment(os, "%dummy", parent_context);
+                       //output_ert_inset(os, "%dummy", parent_context);
                        p.get_token();
                        p.skip_spaces();
                        parent_context.new_paragraph(os);
                }
                else if (p.next_token().cat() == catSpace || p.next_token().cat() == catNewline) {
-                       //handle_comment(os, "%dummy", parent_context);
+                       //output_ert_inset(os, "%dummy", parent_context);
                        p.get_token();
                        p.skip_spaces();
                        // We add a protected space if something real follows
@@ -1198,24 +1186,14 @@ void parse_listings(Parser & p, ostream & os, Context & parent_context, bool in_
        context.layout = &parent_context.textclass.plainLayout();
        string s;
        if (in_line) {
-               s = p.plainCommand('!', '!', "lstinline");
-               context.new_paragraph(os);
-               context.check_layout(os);
+               // set catcodes to verbatim early, just in case.
+               p.setCatcodes(VERBATIM_CATCODES);
+               string delim = p.get_token().asInput();
+               s = p.verbatimStuff(delim);
+//             context.new_paragraph(os);
        } else
-               s = p.plainEnvironment("lstlisting");
-       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
-               if (*it == '\\')
-                       os << "\n\\backslash\n";
-               else if (*it == '\n') {
-                       // avoid adding an empty paragraph at the end
-                       if (it + 1 != et) {
-                               context.new_paragraph(os);
-                               context.check_layout(os);
-                       }
-               } else
-                       os << *it;
-       }
-       context.check_end_layout(os);
+               s = p.verbatimEnvironment("lstlisting");
+       output_ert(os, s, context);
        end_inset(os);
 }
 
@@ -1239,9 +1217,9 @@ void parse_unknown_environment(Parser & p, string const & name, ostream & os,
        bool const new_layout_allowed = parent_context.new_layout_allowed;
        if (specialfont)
                parent_context.new_layout_allowed = false;
-       handle_ert(os, "\\begin{" + name + "}", parent_context);
+       output_ert_inset(os, "\\begin{" + name + "}", parent_context);
        parse_text_snippet(p, os, flags, outer, parent_context);
-       handle_ert(os, "\\end{" + name + "}", parent_context);
+       output_ert_inset(os, "\\end{" + name + "}", parent_context);
        if (specialfont)
                parent_context.new_layout_allowed = new_layout_allowed;
 }
@@ -1435,27 +1413,27 @@ void parse_environment(Parser & p, ostream & os, bool outer,
        }
 
        else if (name == "verbatim") {
-               os << "\n\\end_layout\n\n\\begin_layout Verbatim\n";
-               string const s = p.plainEnvironment("verbatim");
-               string::const_iterator it2 = s.begin();
-               for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
-                       if (*it == '\\')
-                               os << "\\backslash ";
-                       else if (*it == '\n') {
-                               it2 = it + 1;
-                               // avoid adding an empty paragraph at the end
-                               // FIXME: if there are 2 consecutive spaces at the end ignore it
-                               // because LyX will re-add a \n
-                               // This hack must be removed once bug 8049 is fixed!
-                               if ((it + 1 != et) && (it + 2 != et || *it2 != '\n'))
-                                       os << "\n\\end_layout\n\\begin_layout Verbatim\n";
-                       } else
-                               os << *it;
-               }
-               os << "\n\\end_layout\n\n";
+               // FIXME: this should go in the generic code that
+               // handles environments defined in layout file that
+               // have "PassThru 1". However, the code over there is
+               // already too complicated for my taste.
+               parent_context.new_paragraph(os);
+               Context context(true, parent_context.textclass,
+                               &parent_context.textclass[from_ascii("Verbatim")]);
+               string s = p.verbatimEnvironment("verbatim");
+               output_ert(os, s, context);
+               p.skip_spaces();
+       }
+
+       else if (name == "IPA") {
+               eat_whitespace(p, os, parent_context, false);
+               parent_context.check_layout(os);
+               begin_inset(os, "IPA\n");
+               parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
+               end_inset(os);
                p.skip_spaces();
-               // reset to Standard layout
-               os << "\n\\begin_layout Standard\n";
+               preamble.registerAutomaticallyLoadedPackage("tipa");
+               preamble.registerAutomaticallyLoadedPackage("tipx");
        }
 
        else if (name == "CJK") {
@@ -1489,7 +1467,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                bool knownMapping = mapping == preamble.fontCJK();
                if (buggy_encoding || !knownMapping || !where) {
                        parent_context.check_layout(os);
-                       handle_ert(os, "\\begin{" + name + "}{" + encoding + "}{" + mapping + "}",
+                       output_ert_inset(os, "\\begin{" + name + "}{" + encoding + "}{" + mapping + "}",
                                       parent_context);
                        // we must parse the content as verbatim because e.g. JIS can contain
                        // normally invalid characters
@@ -1499,13 +1477,15 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        string const s = p.plainEnvironment("CJK");
                        for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
                                if (*it == '\\')
-                                       handle_ert(os, "\\", parent_context);
+                                       output_ert_inset(os, "\\", parent_context);
                                else if (*it == '$')
-                                       handle_ert(os, "$", parent_context);
+                                       output_ert_inset(os, "$", parent_context);
+                               else if (*it == '\n' && it + 1 != et && s.begin() + 1 != it)
+                                       os << "\n ";
                                else
                                        os << *it;
                        }
-                       handle_ert(os, "\\end{" + name + "}",
+                       output_ert_inset(os, "\\end{" + name + "}",
                                       parent_context);
                } else {
                        string const lang =
@@ -1720,7 +1700,14 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                begin_inset(os, "Flex ");
                os << to_utf8(newinsetlayout->name()) << '\n'
                   << "status collapsed\n";
-               parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
+               if (newinsetlayout->isPassThru()) {
+                       string const arg = p.verbatimEnvironment(name);
+                       Context context(true, parent_context.textclass, 
+                                       &parent_context.textclass.plainLayout(),
+                                       parent_context.layout);
+                       output_ert(os, arg, parent_context);
+               } else
+                       parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
                end_inset(os);
        }
 
@@ -1758,12 +1745,12 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                parse_arguments("\\begin{" + name + "}", arguments, p, os,
                                outer, parent_context);
                if (contents == verbatim)
-                       handle_ert(os, p.verbatimEnvironment(name),
+                       output_ert_inset(os, p.ertEnvironment(name),
                                   parent_context);
                else
                        parse_text_snippet(p, os, FLAG_END, outer,
                                           parent_context);
-               handle_ert(os, "\\end{" + name + "}", parent_context);
+               output_ert_inset(os, "\\end{" + name + "}", parent_context);
                if (specialfont)
                        parent_context.new_layout_allowed = new_layout_allowed;
        }
@@ -1783,7 +1770,7 @@ void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
        LASSERT(t.cat() == catComment, return);
        if (!t.cs().empty()) {
                context.check_layout(os);
-               handle_comment(os, '%' + t.cs(), context);
+               output_ert_inset(os, '%' + t.cs(), context);
                if (p.next_token().cat() == catNewline) {
                        // A newline after a comment line starts a new
                        // paragraph
@@ -1793,7 +1780,7 @@ void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
                                        // done (we might get called recursively)
                                        context.new_paragraph(os);
                        } else
-                               handle_ert(os, "\n", context);
+                               output_ert_inset(os, "\n", context);
                        eat_whitespace(p, os, context, true);
                }
        } else {
@@ -2150,10 +2137,10 @@ void parse_macro(Parser & p, ostream & os, Context & context)
 
                        // followed by number?
                        if (p.next_token().cat() == catOther) {
-                               char c = p.getChar();
-                               paramtext += c;
+                               string s = p.get_token().asInput();
+                               paramtext += s;
                                // number = current arity + 1?
-                               if (c == arity + '0' + 1)
+                               if (s.size() == 1 && s[0] == arity + '0' + 1)
                                        ++arity;
                                else
                                        simple = false;
@@ -2173,7 +2160,7 @@ void parse_macro(Parser & p, ostream & os, Context & context)
                os << "\n\\def" << ert;
                end_inset(os);
        } else
-               handle_ert(os, command + ert, context);
+               output_ert_inset(os, command + ert, context);
 }
 
 
@@ -2207,9 +2194,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
        Layout const * newlayout = 0;
        InsetLayout const * newinsetlayout = 0;
        char const * const * where = 0;
-       // Store the latest bibliographystyle and nocite{*} option
-       // (needed for bibtex inset)
+       // Store the latest bibliographystyle, addcontentslineContent and
+       // nocite{*} option (needed for bibtex inset)
        string btprint;
+       string contentslineContent;
        string bibliographystyle = "default";
        bool const use_natbib = isProvided("natbib");
        bool const use_jurabib = isProvided("jurabib");
@@ -2363,7 +2351,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        else
                                cerr << "Warning: Inserting missing ']' in '"
                                     << s << "'." << endl;
-                       handle_ert(os, s, context);
+                       output_ert_inset(os, s, context);
                }
 
                else if (t.cat() == catLetter) {
@@ -2380,7 +2368,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                for (int i = 1; i < *l && p.next_token().isAlnumASCII(); ++i)
                                        phrase += p.get_token().cs();
                                if (is_known(phrase, known_coded_phrases)) {
-                                       handle_ert(os, phrase, context);
+                                       output_ert_inset(os, phrase, context);
                                        handled = true;
                                        break;
                                } else {
@@ -2404,7 +2392,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        if (context.new_layout_allowed)
                                context.new_paragraph(os);
                        else
-                               handle_ert(os, "\\par ", context);
+                               output_ert_inset(os, "\\par ", context);
                        eat_whitespace(p, os, context, true);
                }
 
@@ -2433,7 +2421,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                             p.next_token().character() == '-'))
                                ; // ignore it in {}`` or -{}-
                        else
-                               handle_ert(os, "{}", context);
+                               output_ert_inset(os, "{}", context);
                        } else if (next.cat() == catEscape &&
                                   is_known(next.cs(), known_quotes) &&
                                   end.cat() == catEnd) {
@@ -2457,17 +2445,17 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        p.get_token();
                                } else {
                                        p.putback();
-                                       handle_ert(os, "{", context);
+                                       output_ert_inset(os, "{", context);
                                        parse_text_snippet(p, os,
                                                        FLAG_BRACE_LAST,
                                                        outer, context);
-                                       handle_ert(os, "}", context);
+                                       output_ert_inset(os, "}", context);
                                }
                        } else if (! context.new_layout_allowed) {
-                               handle_ert(os, "{", context);
+                               output_ert_inset(os, "{", context);
                                parse_text_snippet(p, os, FLAG_BRACE_LAST,
                                                   outer, context);
-                               handle_ert(os, "}", context);
+                               output_ert_inset(os, "}", context);
                        } else if (is_known(next.cs(), known_sizes)) {
                                // next will change the size, so we must
                                // reset it here
@@ -2515,10 +2503,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                           << "\n\\shape "
                                           << context.font.shape << "\n";
                        } else {
-                               handle_ert(os, "{", context);
+                               output_ert_inset(os, "{", context);
                                parse_text_snippet(p, os, FLAG_BRACE_LAST,
                                                   outer, context);
-                               handle_ert(os, "}", context);
+                               output_ert_inset(os, "}", context);
                                }
                        }
                }
@@ -2528,7 +2516,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                return;
                        }
                        cerr << "stray '}' in text\n";
-                       handle_ert(os, "}", context);
+                       output_ert_inset(os, "}", context);
                }
 
                else if (t.cat() == catComment)
@@ -2594,9 +2582,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                // FIXME: Do this in check_layout()!
                                context.has_item = false;
                                if (optarg)
-                                       handle_ert(os, "\\item", context);
+                                       output_ert_inset(os, "\\item", context);
                                else
-                                       handle_ert(os, "\\item ", context);
+                                       output_ert_inset(os, "\\item ", context);
                        }
                        if (optarg) {
                                if (context.layout->labeltype != LABEL_MANUAL) {
@@ -2616,7 +2604,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        // would misinterpret the space as
                                        // item delimiter (bug 7663)
                                        if (contains(s, ' ')) {
-                                               handle_ert(os, s, context);
+                                               output_ert_inset(os, s, context);
                                        } else {
                                                Parser p2(s + ']');
                                                os << parse_text_snippet(p2,
@@ -2639,7 +2627,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        string key = convert_command_inset_arg(p.verbatim_item());
                        if (contains(label, '\\') || contains(key, '\\')) {
                                // LyX can't handle LaTeX commands in labels or keys
-                               handle_ert(os, t.asInput() + '[' + label +
+                               output_ert_inset(os, t.asInput() + '[' + label +
                                               "]{" + p.verbatim_item() + '}',
                                           context);
                        } else {
@@ -2723,7 +2711,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        context.check_layout(os);
                        // FIXME: This is a hack to prevent paragraph
                        // deletion if it is empty. Handle this better!
-                       handle_comment(os,
+                       output_ert_inset(os,
                                "%dummy comment inserted by tex2lyx to "
                                "ensure that this paragraph is not empty",
                                context);
@@ -2759,7 +2747,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        for (; it != en; ++it)
                                                preamble.registerAutomaticallyLoadedPackage(*it);
                                } else
-                                       handle_ert(os,
+                                       output_ert_inset(os,
                                                "\\date{" + p.verbatim_item() + '}',
                                                context);
                        }
@@ -2877,10 +2865,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                // output it as ERT
                                if (p.hasOpt()) {
                                        string opt_arg = convert_command_inset_arg(p.getArg('[', ']'));
-                                       handle_ert(os, t.asInput() + '[' + opt_arg +
+                                       output_ert_inset(os, t.asInput() + '[' + opt_arg +
                                               "]{" + p.verbatim_item() + '}', context);
                                } else
-                                       handle_ert(os, t.asInput() + "{" + p.verbatim_item() + '}', context);
+                                       output_ert_inset(os, t.asInput() + "{" + p.verbatim_item() + '}', context);
                        }
                }
 
@@ -3078,7 +3066,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5")
                                os << s;
                        else
-                               handle_ert(os, "\\ensuremath{" + s + "}",
+                               output_ert_inset(os, "\\ensuremath{" + s + "}",
                                           context);
                }
 
@@ -3087,7 +3075,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                // swallow this
                                skip_spaces_braces(p);
                        } else
-                               handle_ert(os, t.asInput(), context);
+                               output_ert_inset(os, t.asInput(), context);
                }
 
                else if (t.cs() == "tableofcontents" || t.cs() == "lstlistoflistings") {
@@ -3123,7 +3111,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                end_inset(os);
                                p.get_token(); // swallow second arg
                        } else
-                               handle_ert(os, "\\listof{" + name + "}", context);
+                               output_ert_inset(os, "\\listof{" + name + "}", context);
                }
 
                else if ((where = is_known(t.cs(), known_text_font_families)))
@@ -3172,7 +3160,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        preamble.registerAutomaticallyLoadedPackage("color");
                        } else
                                // for custom defined colors
-                               handle_ert(os, t.asInput() + "{" + color + "}", context);
+                               output_ert_inset(os, t.asInput() + "{" + color + "}", context);
                }
 
                else if (t.cs() == "underbar" || t.cs() == "uline") {
@@ -3252,6 +3240,83 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                }
 
+               else if (t.cs() == "textipa") {
+                       context.check_layout(os);
+                       begin_inset(os, "IPA\n");
+                       parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
+                       end_inset(os);
+                       preamble.registerAutomaticallyLoadedPackage("tipa");
+                       preamble.registerAutomaticallyLoadedPackage("tipx");
+               }
+
+               else if (t.cs() == "texttoptiebar" || t.cs() == "textbottomtiebar") {
+                       context.check_layout(os);
+                       begin_inset(os, "IPADeco " + t.cs().substr(4) + "\n");
+                       os << "status open\n";
+                       parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
+                       end_inset(os);
+                       p.skip_spaces();
+               }
+
+               // the TIPA Combining diacritical marks
+               else if (is_known(t.cs(), known_tipa_marks) || t.cs() == "textvertline") {
+                       preamble.registerAutomaticallyLoadedPackage("tipa");
+                       preamble.registerAutomaticallyLoadedPackage("tipx");
+                       context.check_layout(os);
+                       if (t.cs() == "textvertline") {
+                               os << "|";
+                               skip_braces(p);
+                               continue;
+                       }
+                       // try to see whether the string is in unicodesymbols
+                       bool termination;
+                       docstring rem;
+                       string content = trimSpaceAndEol(p.verbatim_item());
+                       string command = t.asInput() + "{" + content + "}";
+                       set<string> req;
+                       docstring s = encodings.fromLaTeXCommand(from_utf8(command),
+                               Encodings::TEXT_CMD | Encodings::MATH_CMD,
+                               termination, rem, &req);
+                       if (!s.empty()) {
+                               if (!rem.empty())
+                                       cerr << "When parsing " << command
+                                            << ", result is " << to_utf8(s)
+                                            << "+" << to_utf8(rem) << endl;
+                               os << content << to_utf8(s);
+                       } else
+                               // we did not find a non-ert version
+                               output_ert_inset(os, command, context);
+               }
+
+               else if (t.cs() == "tone" ) {
+                       context.check_layout(os);
+                       // register the tone package
+                       preamble.registerAutomaticallyLoadedPackage("tone");
+                       string content = trimSpaceAndEol(p.verbatim_item());
+                       string command = t.asInput() + "{" + content + "}";
+                       // some tones can be detected by unicodesymbols, some need special code
+                       if (is_known(content, known_tones)) {
+                               os << "\\IPAChar " << command << "\n";
+                               continue;
+                       }
+                       // try to see whether the string is in unicodesymbols
+                       bool termination;
+                       docstring rem;
+                       set<string> req;
+                       docstring s = encodings.fromLaTeXCommand(from_utf8(command),
+                               Encodings::TEXT_CMD | Encodings::MATH_CMD,
+                               termination, rem, &req);
+                       if (!s.empty()) {
+                               if (!rem.empty())
+                                       cerr << "When parsing " << command
+                                            << ", result is " << to_utf8(s)
+                                                << "+" << to_utf8(rem) << endl;
+                               os << to_utf8(s);
+                       } else
+                               // we did not find a non-ert version
+                               output_ert_inset(os, command, context);
+               }
+
                else if (t.cs() == "phantom" || t.cs() == "hphantom" ||
                             t.cs() == "vphantom") {
                        context.check_layout(os);
@@ -3369,7 +3434,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                preamble.registerAutomaticallyLoadedPackage("varioref");
                                } else {
                                        // LyX does not yet support optional arguments of ref commands
-                                       handle_ert(os, t.asInput() + '[' + opt + "]{" +
+                                       output_ert_inset(os, t.asInput() + '[' + opt + "]{" +
                                               p.verbatim_item() + "}", context);
                                }
                        }
@@ -3560,10 +3625,23 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        end_inset(os);
                }
 
-               else if (t.cs() == "printindex") {
+               else if (t.cs() == "printindex" || t.cs() == "printsubindex") {
                        context.check_layout(os);
-                       begin_command_inset(os, "index_print", "printindex");
-                       os << "type \"idx\"\n";
+                       string commandname = t.cs();
+                       bool star = false;
+                       if (p.next_token().asInput() == "*") {
+                               commandname += "*";
+                               star = true;
+                               p.get_token();
+                       }
+                       begin_command_inset(os, "index_print", commandname);
+                       string const indexname = p.getArg('[', ']');
+                       if (!star) {
+                               if (indexname.empty())
+                                       os << "type \"idx\"\n";
+                               else
+                                       os << "type \"" << indexname << "\"\n";
+                       }
                        end_inset(os);
                        skip_spaces_braces(p);
                        preamble.registerAutomaticallyLoadedPackage("makeidx");
@@ -3730,7 +3808,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                                  context, "\\lang",
                                                                  context.font.language, lang);
                                } else
-                                       handle_ert(os, t.asInput() + langopts, context);
+                                       output_ert_inset(os, t.asInput() + langopts, context);
                        } else {
                                lang = preamble.polyglossia2lyx(t.cs().substr(4, string::npos));
                                parse_text_attributes(p, os, FLAG_ITEM, outer,
@@ -3810,31 +3888,27 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        os << '"';
                                        skip_braces(p);
                                } else {
-                                       handle_ert(os, "\\char`", context);
+                                       output_ert_inset(os, "\\char`", context);
                                }
                        } else {
-                               handle_ert(os, "\\char", context);
+                               output_ert_inset(os, "\\char", context);
                        }
                }
 
                else if (t.cs() == "verb") {
                        context.check_layout(os);
-                       char const delimiter = p.next_token().character();
-                       // \verb is special: The usual escaping rules do not
-                       // apply, e.g. "\verb+\+" is valid and denotes a single
-                       // backslash (bug #4468). Therefore we do not allow
-                       // escaping in getArg().
-                       string const arg = p.getArg(delimiter, delimiter, false);
-                       ostringstream oss;
-                       oss << "\\verb" << delimiter << arg << delimiter;
-                       handle_ert(os, oss.str(), context);
+                       // set catcodes to verbatim early, just in case.
+                       p.setCatcodes(VERBATIM_CATCODES);
+                       string delim = p.get_token().asInput();
+                       string const arg = p.verbatimStuff(delim);
+                       output_ert_inset(os, "\\verb" + delim + arg + delim, context);
                }
 
                // Problem: \= creates a tabstop inside the tabbing environment
                // and else an accent. In the latter case we really would want
                // \={o} instead of \= o.
                else if (t.cs() == "=" && (flags & FLAG_TABBING))
-                       handle_ert(os, t.asInput(), context);
+                       output_ert_inset(os, t.asInput(), context);
 
                // accents (see Table 6 in Comprehensive LaTeX Symbol List)
                else if (t.cs().size() == 1
@@ -3860,19 +3934,19 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        preamble.registerAutomaticallyLoadedPackage(*it);
                        } else
                                // we did not find a non-ert version
-                               handle_ert(os, command, context);
+                               output_ert_inset(os, command, context);
                }
 
                else if (t.cs() == "\\") {
                        context.check_layout(os);
                        if (p.hasOpt())
-                               handle_ert(os, "\\\\" + p.getOpt(), context);
+                               output_ert_inset(os, "\\\\" + p.getOpt(), context);
                        else if (p.next_token().asInput() == "*") {
                                p.get_token();
                                // getOpt() eats the following space if there
                                // is no optional argument, but that is OK
                                // here since it has no effect in the output.
-                               handle_ert(os, "\\\\*" + p.getOpt(), context);
+                               output_ert_inset(os, "\\\\*" + p.getOpt(), context);
                        }
                        else {
                                begin_inset(os, "Newline newline");
@@ -4012,8 +4086,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "bibliographystyle") {
                        // store new bibliographystyle
                        bibliographystyle = p.verbatim_item();
-                       // If any other command than \bibliography and
-                       // \nocite{*} follows, we need to output the style
+                       // If any other command than \bibliography, \addcontentsline
+                       // and \nocite{*} follows, we need to output the style
                        // (because it might be used by that command).
                        // Otherwise, it will automatically be output by LyX.
                        p.pushPosition();
@@ -4028,18 +4102,53 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                continue;
                                } else if (t2.cs() == "bibliography")
                                        output = false;
+                               else if (t2.cs() == "phantomsection") {
+                                       output = false;
+                                       continue;
+                               }
+                               else if (t2.cs() == "addcontentsline") {
+                                       // get the 3 arguments of \addcontentsline
+                                       p.getArg('{', '}');
+                                       p.getArg('{', '}');
+                                       contentslineContent = p.getArg('{', '}');
+                                       // if the last argument is not \refname we must output
+                                       if (contentslineContent == "\\refname")
+                                               output = false;
+                               }
                                break;
                        }
                        p.popPosition();
                        if (output) {
-                               handle_ert(os,
+                               output_ert_inset(os,
                                        "\\bibliographystyle{" + bibliographystyle + '}',
                                        context);
                        }
                }
 
+               else if (t.cs() == "phantomsection") {
+                       // we only support this if it occurs between
+                       // \bibliographystyle and \bibliography
+                       if (bibliographystyle.empty())
+                               output_ert_inset(os, "\\phantomsection", context);
+               }
+
+               else if (t.cs() == "addcontentsline") {
+                       context.check_layout(os);
+                       // get the 3 arguments of \addcontentsline
+                       string const one = p.getArg('{', '}');
+                       string const two = p.getArg('{', '}');
+                       string const three = p.getArg('{', '}');
+                       // only if it is a \refname, we support if for the bibtex inset
+                       if (contentslineContent != "\\refname") {
+                               output_ert_inset(os,
+                                       "\\addcontentsline{" + one + "}{" + two + "}{"+ three + '}',
+                                       context);
+                       }
+               }
+
                else if (t.cs() == "bibliography") {
                        context.check_layout(os);
+                       string BibOpts;
                        begin_command_inset(os, "bibtex", "bibtex");
                        if (!btprint.empty()) {
                                os << "btprint " << '"' << "btPrintAll" << '"' << "\n";
@@ -4048,9 +4157,23 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                btprint.clear();
                        }
                        os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n";
+                       // Do we have addcontentsline?
+                       if (contentslineContent == "\\refname") {
+                               BibOpts = "bibtotoc";
+                               // clear string because next BibTeX inset can be without addcontentsline
+                               contentslineContent.clear();
+                       }
                        // Do we have a bibliographystyle set?
-                       if (!bibliographystyle.empty())
-                               os << "options " << '"' << bibliographystyle << '"' << "\n";
+                       if (!bibliographystyle.empty()) {
+                               if (BibOpts.empty())
+                                       BibOpts = bibliographystyle;
+                               else
+                                       BibOpts = BibOpts + ',' + bibliographystyle;
+                               // clear it because each bibtex entry has its style
+                               // and we need an empty string to handle \phantomsection
+                               bibliographystyle.clear();
+                       }
+                       os << "options " << '"' << BibOpts << '"' << "\n";
                        end_inset(os);
                }
 
@@ -4095,9 +4218,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                arg += p.getFullParentheseArg();
                                arg += p.getFullOpt();
                                eat_whitespace(p, os, context, false);
-                               handle_ert(os, arg + '{', context);
+                               output_ert_inset(os, arg + '{', context);
                                parse_text(p, os, FLAG_ITEM, outer, context);
-                               handle_ert(os, "}", context);
+                               output_ert_inset(os, "}", context);
                        } else {
                                string special = p.getFullOpt();
                                special += p.getOpt();
@@ -4107,9 +4230,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                        context, t.cs(), special);
                                else {
                                        eat_whitespace(p, os, context, false);
-                                       handle_ert(os, "\\framebox{", context);
+                                       output_ert_inset(os, "\\framebox{", context);
                                        parse_text(p, os, FLAG_ITEM, outer, context);
-                                       handle_ert(os, "}", context);
+                                       output_ert_inset(os, "}", context);
                                }
                        }
                }
@@ -4123,9 +4246,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                arg += p.getFullParentheseArg();
                                arg += p.getFullOpt();
                                eat_whitespace(p, os, context, false);
-                               handle_ert(os, arg + '{', context);
+                               output_ert_inset(os, arg + '{', context);
                                parse_text(p, os, FLAG_ITEM, outer, context);
-                               handle_ert(os, "}", context);
+                               output_ert_inset(os, "}", context);
                        } else
                                //the syntax is: \makebox[width][position]{content}
                                parse_box(p, os, 0, FLAG_ITEM, outer, context,
@@ -4204,7 +4327,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                            t.cs() == "providecommand" ||
                            t.cs() == "providecommandx" ||
                            name[name.length()-1] == '*')
-                               handle_ert(os, ert, context);
+                               output_ert_inset(os, ert, context);
                        else {
                                context.check_layout(os);
                                begin_inset(os, "FormulaMacro");
@@ -4241,7 +4364,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        CommandMap::iterator it = known_commands.find(command);
                        if (it != known_commands.end())
                                known_commands[t.asInput()] = it->second;
-                       handle_ert(os, ert, context);
+                       output_ert_inset(os, ert, context);
                }
 
                else if (t.cs() == "hspace" || t.cs() == "vspace") {
@@ -4340,13 +4463,13 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        name += '*';
                                if (valid) {
                                        if (value == 1.0)
-                                               handle_ert(os, name + '{' + unit + '}', context);
+                                               output_ert_inset(os, name + '{' + unit + '}', context);
                                        else if (value == -1.0)
-                                               handle_ert(os, name + "{-" + unit + '}', context);
+                                               output_ert_inset(os, name + "{-" + unit + '}', context);
                                        else
-                                               handle_ert(os, name + '{' + valstring + unit + '}', context);
+                                               output_ert_inset(os, name + '{' + valstring + unit + '}', context);
                                } else
-                                       handle_ert(os, name + '{' + length + '}', context);
+                                       output_ert_inset(os, name + '{' + length + '}', context);
                        }
                }
 
@@ -4357,7 +4480,20 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        begin_inset(os, "Flex ");
                        os << to_utf8(newinsetlayout->name()) << '\n'
                           << "status collapsed\n";
-                       parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
+                       if (newinsetlayout->isPassThru()) {
+                               // set catcodes to verbatim early, just in case.
+                               p.setCatcodes(VERBATIM_CATCODES);
+                               string delim = p.get_token().asInput();
+                               if (delim != "{")
+                                       cerr << "Warning: bad delimiter for command " << t.asInput() << endl;
+                               string const arg = p.verbatimStuff("}");
+                               Context newcontext(true, context.textclass);
+                               if (newinsetlayout->forcePlainLayout())
+                                       newcontext.layout = &context.textclass.plainLayout();
+                               output_ert(os, arg, newcontext);
+                       } else
+                               
+                               parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
                        end_inset(os);
                }
 
@@ -4475,7 +4611,72 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        bool termination;
                        docstring rem;
                        set<string> req;
-                       docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
+                       string name = t.asInput();
+                       // handle some TIPA special characters
+                       if (name == "\\textglobfall") {
+                               name = "End";
+                               skip_braces(p);
+                       }
+                       if (name == "\\textdoublevertline") {
+                               name = "\\textbardbl";
+                               skip_braces(p);
+                       }
+                       if (name == "\\!" ) {
+                               if (p.next_token().asInput() == "b") {
+                                       p.get_token();  // eat 'b'
+                                       name = "\\texthtb";
+                                       skip_braces(p);
+                               }
+                               if (p.next_token().asInput() == "d") {
+                                       p.get_token();
+                                       name = "\\texthtd";
+                                       skip_braces(p);
+                               }
+                               if (p.next_token().asInput() == "g") {
+                                       p.get_token();
+                                       name = "\\texthtg";
+                                       skip_braces(p);
+                               }
+                               if (p.next_token().asInput() == "G") {
+                                       p.get_token();
+                                       name = "\\texthtscg";
+                                       skip_braces(p);
+                               }
+                               if (p.next_token().asInput() == "j") {
+                                       p.get_token();
+                                       name = "\\texthtbardotlessj";
+                                       skip_braces(p);
+                               }
+                               if (p.next_token().asInput() == "o") {
+                                       p.get_token();
+                                       name = "\\textbullseye";
+                                       skip_braces(p);
+                               }
+                       }
+                       if (name == "\\*" ) {
+                               if (p.next_token().asInput() == "k") {
+                                       p.get_token();
+                                       name = "\\textturnk";
+                                       skip_braces(p);
+                               }
+                               if (p.next_token().asInput() == "r") {
+                                       p.get_token();  // eat 'b'
+                                       name = "\\textturnr";
+                                       skip_braces(p);
+                               }                               
+                               if (p.next_token().asInput() == "t") {
+                                       p.get_token();
+                                       name = "\\textturnt";
+                                       skip_braces(p);
+                               }
+                               if (p.next_token().asInput() == "w") {
+                                       p.get_token();
+                                       name = "\\textturnw";
+                                       skip_braces(p);
+                               }                               
+                       }
+                       // now get the character from unicodesymbols
+                       docstring s = encodings.fromLaTeXCommand(from_utf8(name),
                                        Encodings::TEXT_CMD, termination, rem, &req);
                        if (!s.empty()) {
                                if (!rem.empty())
@@ -4500,7 +4701,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                z = p.verbatim_item();
                        }
                        cerr << "found ERT: " << s << endl;
-                       handle_ert(os, s + ' ', context);
+                       output_ert_inset(os, s + ' ', context);
                        */
                        else {
                                string name = t.asInput();
@@ -4510,7 +4711,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        name += '*';
                                }
                                if (!parse_command(name, p, os, outer, context))
-                                       handle_ert(os, name, context);
+                                       output_ert_inset(os, name, context);
                        }
                }
 
@@ -4574,7 +4775,7 @@ string guessLanguage(Parser & p, string const & lang)
                        p.setEncoding(encoding, Encoding::CJK);
                else
                        p.setEncoding("UTF-8");
-               string const text = p.verbatimEnvironment("CJK");
+               string const text = p.ertEnvironment("CJK");
                p.setEncoding(encoding_old);
                p.skip_spaces();
                if (!where) {