]> git.lyx.org Git - lyx.git/blobdiff - src/tex2lyx/text.cpp
Make undo action no-ops when the buffer is read-only
[lyx.git] / src / tex2lyx / text.cpp
index 919f454c66a6c652d9e082527747eb7fdefbd234..76d11a7cb8830ff8e80fc0c6793a357472c3fc35 100644 (file)
@@ -21,7 +21,6 @@
 #include "FloatList.h"
 #include "LaTeXPackages.h"
 #include "Layout.h"
-#include "Length.h"
 #include "Preamble.h"
 
 #include "insets/ExternalTemplate.h"
@@ -30,6 +29,7 @@
 #include "support/convert.h"
 #include "support/FileName.h"
 #include "support/filetools.h"
+#include "support/Length.h"
 #include "support/lstrings.h"
 #include "support/lyxtime.h"
 
@@ -47,15 +47,15 @@ namespace lyx {
 
 namespace {
 
-void output_arguments(ostream &, Parser &, bool, bool, string, Context &,
+void output_arguments(ostream &, Parser &, bool, bool, const string &, Context &,
                       Layout::LaTeXArgMap const &);
 
 }
 
 
 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
-               Context const & context, InsetLayout const * layout,
-               string const rdelim)
+               Context & context, InsetLayout const * layout,
+               string const rdelim)
 {
        bool const forcePlainLayout =
                layout ? layout->forcePlainLayout() : false;
@@ -64,6 +64,10 @@ void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
                newcontext.layout = &context.textclass.plainLayout();
        else
                newcontext.font = context.font;
+       // Inherit commands to pass through
+       newcontext.pass_thru_cmds = context.pass_thru_cmds;
+       // and table cell
+       newcontext.in_table_cell = context.in_table_cell;
        if (layout)
                output_arguments(os, p, outer, false, string(), newcontext,
                                 layout->latexargs());
@@ -79,6 +83,7 @@ void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
                output_arguments(os, p, outer, false, "post", newcontext,
                                 layout->postcommandargs());
        newcontext.check_end_layout(os);
+       context.cell_align = newcontext.cell_align;
 }
 
 
@@ -86,14 +91,15 @@ namespace {
 
 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
                Context const & context, string const & name,
-               string const rdelim = string())
+               string const rdelim = string())
 {
        InsetLayout const * layout = 0;
        DocumentClass::InsetLayouts::const_iterator it =
                context.textclass.insetLayouts().find(from_ascii(name));
        if (it != context.textclass.insetLayouts().end())
                layout = &(it->second);
-       parse_text_in_inset(p, os, flags, outer, context, layout, rdelim);
+       Context newcontext = context;
+       parse_text_in_inset(p, os, flags, outer, newcontext, layout, rdelim);
 }
 
 /// parses a paragraph snippet, useful for example for \\emph{...}
@@ -316,18 +322,26 @@ char const * const known_pdftex_graphics_formats[] = {"png", "pdf", "jpg",
 char const * const known_tex_extensions[] = {"tex", 0};
 
 /// spaces known by InsetSpace
-char const * const known_spaces[] = { " ", "space", ",",
-"thinspace", "quad", "qquad", "enspace", "enskip",
-"negthinspace", "negmedspace", "negthickspace", "textvisiblespace",
-"hfill", "dotfill", "hrulefill", "leftarrowfill", "rightarrowfill",
-"upbracefill", "downbracefill", 0};
+char const * const known_spaces[] = { " ", "space", 
+",", "thinspace",//                                   \\, = \\thinspace
+"quad", "qquad", "enspace", "enskip",
+";", ">", "medspace",//                               \\; = \\> = \\medspace
+":", "thickspace",//                                  \\: = \\thickspace
+"!", "negthinspace",//                                \\! = \\negthinspace
+"negmedspace", "negthickspace",
+"textvisiblespace", "hfill", "dotfill", "hrulefill", "leftarrowfill",
+"rightarrowfill", "upbracefill", "downbracefill", 0};
 
 /// the same as known_spaces with .lyx names
 char const * const known_coded_spaces[] = { "space{}", "space{}",
-"thinspace{}", "thinspace{}", "quad{}", "qquad{}", "enspace{}", "enskip{}",
-"negthinspace{}", "negmedspace{}", "negthickspace{}", "textvisiblespace{}",
-"hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}", "rightarrowfill{}",
-"upbracefill{}", "downbracefill{}", 0};
+"thinspace{}", "thinspace{}",
+"quad{}", "qquad{}", "enspace{}", "enskip{}",
+"medspace{}", "medspace{}", "medspace{}",
+"thickspace{}", "thickspace{}",
+"negthinspace{}", "negthinspace{}",
+"negmedspace{}", "negthickspace{}",
+"textvisiblespace{}", "hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}",
+"rightarrowfill{}", "upbracefill{}", "downbracefill{}", 0};
 
 /// known TIPA combining diacritical marks
 char const * const known_tipa_marks[] = {"textsubwedge", "textsubumlaut",
@@ -459,7 +473,7 @@ bool translate_len(string const & length, string & valstring, string & unit)
 
 /// If we have ambiguous quotation marks, make a smart guess
 /// based on main quote style
-string guessQuoteStyle(string in, bool const opening)
+string guessQuoteStyle(string const & in, bool const opening)
 {
        string res = in;
        if (prefixIs(in, "qr")) {// straight quote
@@ -481,6 +495,8 @@ string guessQuoteStyle(string in, bool const opening)
                        res = "brs";
                else if (preamble.quotesStyle() == "french")
                        res = "frs";
+               else if (preamble.quotesStyle() == "hungarian")
+                       res = "hrd";
                else if (preamble.quotesStyle() == "swedish")
                        res = opening ? "sld" : "srd";
        } else if (in == "els") {// `
@@ -502,6 +518,8 @@ string guessQuoteStyle(string in, bool const opening)
                        res = "fld";
                else if (preamble.quotesStyle() == "russian")
                        res = "rld";
+               else if (preamble.quotesStyle() == "hungarian")
+                       res = "hrs";
        } else if (in == "ald") {// <<
                if (preamble.quotesStyle() == "swiss")
                        res = "crd";
@@ -509,6 +527,8 @@ string guessQuoteStyle(string in, bool const opening)
                        res = "frd";
                else if (preamble.quotesStyle() == "russian")
                        res = "rrd";
+               else if (preamble.quotesStyle() == "hungarian")
+                       res = "hls";
        } else if (in == "ars") {// >
                if (preamble.quotesStyle() == "swiss")
                        res = "cls";
@@ -518,6 +538,8 @@ string guessQuoteStyle(string in, bool const opening)
        } else if (in == "gld") {// ,,
                if (preamble.quotesStyle() == "polish")
                        res = "pld";
+               else if (preamble.quotesStyle() == "hungarian")
+                       res = "hld";
                else if (preamble.quotesStyle() == "russian")
                        res = "rls";
        } else if (in == "gls") {// ,
@@ -528,6 +550,35 @@ string guessQuoteStyle(string in, bool const opening)
 }
 
 
+string const fromPolyglossiaEnvironment(string const & s)
+{
+       // Since \arabic is taken by the LaTeX kernel,
+       // the Arabic polyglossia environment is upcased
+       if (s == "Arabic")
+               return "arabic";
+       else
+               return s;
+}
+
+
+string uncapitalize(string const & s)
+{
+       docstring in = from_ascii(s);
+       char_type t = lowercase(s[0]);
+       in[0] = t;
+       return to_ascii(in);
+}
+
+
+bool isCapitalized(string const & s)
+{
+       docstring in = from_ascii(s);
+       char_type t = uppercase(s[0]);
+       in[0] = t;
+       return to_ascii(in) == s;
+}
+
+
 } // namespace
 
 
@@ -657,7 +708,7 @@ pair<bool, string> convert_latexed_command_inset_arg(string s)
 
 /// try to convert \p s to a valid InsetCommand argument
 /// without trying to recode macros.
-string convert_literate_command_inset_arg(string s)
+string convert_literate_command_inset_arg(string const & s)
 {
        // LyX cannot handle newlines in a latex command
        return subst(s, "\n", " ");
@@ -666,14 +717,14 @@ string convert_literate_command_inset_arg(string s)
 void output_ert(ostream & os, string const & s, Context & context)
 {
        context.check_layout(os);
-       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
-               if (*it == '\\')
+       for (char const c : s) {
+               if (c == '\\')
                        os << "\n\\backslash\n";
-               else if (*it == '\n') {
+               else if (c == '\n') {
                        context.new_paragraph(os);
                        context.check_layout(os);
                } else
-                       os << *it;
+                       os << c;
        }
        context.check_end_layout(os);
 }
@@ -760,7 +811,7 @@ void skip_spaces_braces(Parser & p, bool keepws = false)
 }
 
 
-void output_arguments(ostream & os, Parser & p, bool outer, bool need_layout, string const prefix,
+void output_arguments(ostream & os, Parser & p, bool outer, bool need_layout, string const prefix,
                       Context & context, Layout::LaTeXArgMap const & latexargs)
 {
        if (context.layout->latextype != LATEX_ITEM_ENVIRONMENT || !prefix.empty()) {
@@ -895,7 +946,7 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
  *
  * 2. could be used to suppress as many spaces as possible. This has two effects:
  * - Reimporting LyX generated LaTeX files changes almost no whitespace
- * - Superflous whitespace from non LyX generated LaTeX files is removed.
+ * - Superfluous whitespace from non LyX generated LaTeX files is removed.
  * The drawback is that the logic inside the function becomes
  * complicated, and that is the reason why it is not implemented.
  */
@@ -1281,7 +1332,7 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
                        os << "use_makebox " << (inner_type == "makebox") << '\n';
                if (outer_type == "mbox" || (outer_type == "fbox" && inner_type.empty()))
                        os << "width \"\"\n";
-               // for values like "1.5\width" LyX uses "1.5in" as width ad sets "width" as sepecial
+               // for values like "1.5\width" LyX uses "1.5in" as width and sets "width" as special
                else if (contains(width_unit, '\\'))
                        os << "width \"" << width_value << "in" << "\"\n";
                else
@@ -1566,6 +1617,14 @@ void parse_unknown_environment(Parser & p, string const & name, ostream & os,
        if (specialfont)
                parent_context.new_layout_allowed = false;
        output_ert_inset(os, "\\begin{" + name + "}", parent_context);
+       // Try to handle options: Look if we have optional arguments,
+       // and if so, put the brackets in ERT.
+       while (p.hasOpt()) {
+               p.get_token(); // eat '['
+               output_ert_inset(os, "[", parent_context);
+               os << parse_text_snippet(p, FLAG_BRACK_LAST, outer, parent_context);
+               output_ert_inset(os, "]", parent_context);
+       }
        parse_text_snippet(p, os, flags, outer, parent_context);
        output_ert_inset(os, "\\end{" + name + "}", parent_context);
        if (specialfont)
@@ -1602,16 +1661,22 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        break;
                }
 
-               if (is_known(name, preamble.polyglossia_languages)) {
+               // We need to use fromPolyglossiaEnvironment due to Arabic > arabic
+               if (is_known(fromPolyglossiaEnvironment(name), preamble.polyglossia_languages)) {
                        // We must begin a new paragraph if not already done
-                       if (! parent_context.atParagraphStart()) {
+                       if (!parent_context.atParagraphStart()) {
                                parent_context.check_end_layout(os);
                                parent_context.new_paragraph(os);
                        }
-                       // save the language in the context so that it is
+                       // store previous language because we must reset it at the end
+                       string const lang_old = parent_context.font.language;
+                       // save new language in context so that it is
                        // handled by parse_text
-                       parent_context.font.language = preamble.polyglossia2lyx(name);
+                       parent_context.font.language =
+                               preamble.polyglossia2lyx(fromPolyglossiaEnvironment(name));
                        parse_text(p, os, FLAG_END, outer, parent_context);
+                       // reset previous language
+                       parent_context.font.language = lang_old;
                        // Just in case the environment is empty
                        parent_context.extra_stuff.erase();
                        // We must begin a new paragraph to reset the language
@@ -1674,7 +1739,13 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        os << "wide " << convert<string>(is_starred)
                           << "\nsideways false"
                           << "\nstatus open\n\n";
+                       set<string> pass_thru_cmds = parent_context.pass_thru_cmds;
+                       if (unstarred_name == "algorithm")
+                               // in algorithm, \; has special meaning
+                               parent_context.pass_thru_cmds.insert(";");
                        parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
+                       if (unstarred_name == "algorithm")
+                               parent_context.pass_thru_cmds = pass_thru_cmds;
                        end_inset(os);
                        // We don't need really a new paragraph, but
                        // we must make sure that the next item gets a \begin_layout.
@@ -1810,7 +1881,13 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        eat_whitespace(p, os, parent_context, false);
                        parent_context.check_layout(os);
                        begin_inset(os, "IPA\n");
+                       set<string> pass_thru_cmds = parent_context.pass_thru_cmds;
+                       // These commands have special meanings in IPA
+                       parent_context.pass_thru_cmds.insert("!");
+                       parent_context.pass_thru_cmds.insert(";");
+                       parent_context.pass_thru_cmds.insert(":");
                        parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
+                       parent_context.pass_thru_cmds = pass_thru_cmds;
                        end_inset(os);
                        p.skip_spaces();
                        preamble.registerAutomaticallyLoadedPackage("tipa");
@@ -2261,6 +2338,9 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        }
                        switch (context.layout->latextype) {
                        case  LATEX_LIST_ENVIRONMENT:
+                               context.in_list_preamble =
+                                       !context.layout->listpreamble().empty()
+                                       && p.hasListPreamble(context.layout->itemcommand());
                                context.add_par_extra_stuff("\\labelwidthstring "
                                                            + p.verbatim_item() + '\n');
                                p.skip_spaces();
@@ -2284,11 +2364,20 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                                output_arguments(os, p, outer, false, string(), context,
                                                 context.layout->latexargs());
                        else if (context.layout->latextype == LATEX_ITEM_ENVIRONMENT) {
+                               context.in_list_preamble =
+                                       !context.layout->listpreamble().empty()
+                                       && p.hasListPreamble(context.layout->itemcommand());
                                ostringstream oss;
                                output_arguments(oss, p, outer, false, string(), context,
                                                 context.layout->latexargs());
                                context.list_extra_stuff = oss.str();
                        }
+                       if (context.in_list_preamble) {
+                               // Collect the stuff between \begin and first \item
+                               context.list_preamble =
+                                       parse_text_snippet(p, FLAG_END, outer, context);
+                               context.in_list_preamble = false;
+                       }
                        parse_text(p, os, FLAG_END, outer, context);
                        if (context.layout->latextype == LATEX_ENVIRONMENT)
                                output_arguments(os, p, outer, false, "post", context,
@@ -2304,7 +2393,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                        p.skip_spaces();
                        if (!preamble.titleLayoutFound())
                                preamble.titleLayoutFound(newlayout->intitle);
-                       set<string> const & req = newlayout->requires();
+                       set<string> const & req = newlayout->required();
                        set<string>::const_iterator it = req.begin();
                        set<string>::const_iterator en = req.end();
                        for (; it != en; ++it)
@@ -2351,7 +2440,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 
                if (known_environments.find(name) != known_environments.end()) {
                        vector<ArgumentType> arguments = known_environments[name];
-                       // The last "argument" denotes wether we may translate the
+                       // The last "argument" denotes whether we may translate the
                        // environment contents to LyX
                        // The default required if no argument is given makes us
                        // compatible with the reLyXre environment.
@@ -2481,7 +2570,7 @@ void get_cite_arguments(Parser & p, bool natbibOrder,
 }
 
 
-void copy_file(FileName const & src, string dstname)
+void copy_file(FileName const & src, string const & dstname)
 {
        if (!copyFiles())
                return;
@@ -2812,7 +2901,7 @@ void fix_child_filename(string & name)
 
 
 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
-               Context & context, string const rdelim)
+               Context & context, string const rdelim)
 {
        Layout const * newlayout = 0;
        InsetLayout const * newinsetlayout = 0;
@@ -2868,6 +2957,13 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                debugToken(cerr, t, flags);
 #endif
 
+               if (context.in_list_preamble
+                   && p.next_token().cs() == context.layout->itemcommand()) {
+                       // We are parsing a list preamble. End before first \item.
+                       flags |= FLAG_LEAVE;
+                       context.in_list_preamble = false;
+               }
+
                if (flags & FLAG_ITEM) {
                        if (t.cat() == catSpace)
                                continue;
@@ -3312,6 +3408,16 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        if (context.layout->labeltype != LABEL_MANUAL)
                                output_arguments(os, p, outer, false, "item", context,
                                                 context.layout->itemargs());
+                       if (!context.list_preamble.empty()) {
+                               // We have a list preamble. Output it here.
+                               begin_inset(os, "Argument listpreamble:1");
+                               os << "\nstatus collapsed\n\n"
+                                  << "\\begin_layout Plain Layout\n\n"
+                                  << rtrim(context.list_preamble)
+                                  << "\n\\end_layout";
+                               end_inset(os);
+                               context.list_preamble.clear();
+                       }
                        if (!context.list_extra_stuff.empty()) {
                                os << context.list_extra_stuff;
                                context.list_extra_stuff.clear();
@@ -3465,7 +3571,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        parse_text_snippet(p, os, FLAG_ITEM, outer, context);
                                        if (!preamble.titleLayoutFound())
                                                preamble.titleLayoutFound(newlayout->intitle);
-                                       set<string> const & req = newlayout->requires();
+                                       set<string> const & req = newlayout->required();
                                        set<string>::const_iterator it = req.begin();
                                        set<string>::const_iterator en = req.end();
                                        for (; it != en; ++it)
@@ -3491,13 +3597,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.skip_spaces();
                        if (!preamble.titleLayoutFound())
                                preamble.titleLayoutFound(newlayout->intitle);
-                       set<string> const & req = newlayout->requires();
+                       set<string> const & req = newlayout->required();
                        for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
                                preamble.registerAutomaticallyLoadedPackage(*it);
                        continue;
                }
 
-
                // Starred section headings
                // Must attempt to parse "Section*" before "Section".
                if ((p.next_token().asInput() == "*") &&
@@ -3509,7 +3614,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.skip_spaces();
                        if (!preamble.titleLayoutFound())
                                preamble.titleLayoutFound(newlayout->intitle);
-                       set<string> const & req = newlayout->requires();
+                       set<string> const & req = newlayout->required();
                        for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
                                preamble.registerAutomaticallyLoadedPackage(*it);
                        continue;
@@ -3523,7 +3628,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.skip_spaces();
                        if (!preamble.titleLayoutFound())
                                preamble.titleLayoutFound(newlayout->intitle);
-                       set<string> const & req = newlayout->requires();
+                       set<string> const & req = newlayout->required();
                        for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
                                preamble.registerAutomaticallyLoadedPackage(*it);
                        continue;
@@ -3666,9 +3771,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        if (opts.find("width") != opts.end())
                                os << "\twidth "
                                   << translate_len(opts["width"]) << '\n';
-                       if (opts.find("height") != opts.end())
+                       if (opts.find("totalheight") != opts.end())
                                os << "\theight "
-                                  << translate_len(opts["height"]) << '\n';
+                                  << translate_len(opts["totalheight"]) << '\n';
                        if (opts.find("scale") != opts.end()) {
                                istringstream iss(opts["scale"]);
                                double val;
@@ -3684,7 +3789,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                vector<string>::const_iterator s =
                                        find(keys.begin(), keys.end(), "width");
                                if (s == keys.end())
-                                       s = find(keys.begin(), keys.end(), "height");
+                                       s = find(keys.begin(), keys.end(), "totalheight");
                                if (s == keys.end())
                                        s = find(keys.begin(), keys.end(), "scale");
                                if (s != keys.end() && distance(s, a) > 0)
@@ -3745,8 +3850,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                special << "trim,";
                        if (opts.find("viewport") != opts.end())
                                special << "viewport=" << opts["viewport"] << ',';
-                       if (opts.find("totalheight") != opts.end())
-                               special << "totalheight=" << opts["totalheight"] << ',';
+                       if (opts.find("height") != opts.end())
+                               special << "height=" << opts["height"] << ',';
                        if (opts.find("type") != opts.end())
                                special << "type=" << opts["type"] << ',';
                        if (opts.find("ext") != opts.end())
@@ -3852,18 +3957,29 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                if (t.cs() == "listof") {
                        p.skip_spaces(true);
-                       string const name = p.get_token().cs();
+                       string const name = p.verbatim_item();
                        if (context.textclass.floats().typeExist(name)) {
                                context.check_layout(os);
                                begin_inset(os, "FloatList ");
                                os << name << "\n";
                                end_inset(os);
-                               p.get_token(); // swallow second arg
+                               p.verbatim_item(); // swallow second arg
                        } else
                                output_ert_inset(os, "\\listof{" + name + "}", context);
                        continue;
                }
 
+               if (t.cs() == "theendnotes"
+                  || (t.cs() == "printendnotes"
+                      && p.next_token().asInput() != "*"
+                      && !p.hasOpt())) {
+                       context.check_layout(os);
+                       begin_inset(os, "FloatList endnote\n");
+                       end_inset(os);
+                       skip_spaces_braces(p);
+                       continue;
+               }
+
                if ((where = is_known(t.cs(), known_text_font_families))) {
                        parse_text_attributes(p, os, FLAG_ITEM, outer,
                                context, "\\family", context.font.family,
@@ -3973,11 +4089,16 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        continue;
                }
 
-               if (t.cs() == "lyxadded" || t.cs() == "lyxdeleted") {
+               if (t.cs() == "lyxadded" || t.cs() == "lyxdeleted" || t.cs() == "lyxobjdeleted"
+                   || t.cs() == "lyxdisplayobjdeleted" || t.cs() == "lyxudisplayobjdeleted") {
                        context.check_layout(os);
+                       string initials;
+                       if (p.hasOpt()) {
+                               initials = p.getArg('[', ']');
+                       }
                        string name = p.getArg('{', '}');
                        string localtime = p.getArg('{', '}');
-                       preamble.registerAuthor(name);
+                       preamble.registerAuthor(name, initials);
                        Author const & author = preamble.getAuthor(name);
                        // from_asctime_utc() will fail if LyX decides to output the
                        // time in the text language.
@@ -3993,7 +4114,6 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                os << "\n\\change_deleted ";
                        os << author.bufferId() << ' ' << ptime << '\n';
                        parse_text_snippet(p, os, FLAG_ITEM, outer, context);
-                       bool dvipost    = LaTeXPackages::isAvailable("dvipost");
                        bool xcolorulem = LaTeXPackages::isAvailable("ulem") &&
                                          LaTeXPackages::isAvailable("xcolor");
                        // No need to test for luatex, since luatex comes in
@@ -4006,9 +4126,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        preamble.registerAutomaticallyLoadedPackage("pdfcolmk");
                                }
                        } else {
-                               if (dvipost) {
-                                       preamble.registerAutomaticallyLoadedPackage("dvipost");
-                               } else if (xcolorulem) {
+                               if (xcolorulem) {
                                        preamble.registerAutomaticallyLoadedPackage("ulem");
                                        preamble.registerAutomaticallyLoadedPackage("xcolor");
                                }
@@ -4021,7 +4139,13 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        begin_inset(os, "IPA\n");
                        bool merging_hyphens_allowed = context.merging_hyphens_allowed;
                        context.merging_hyphens_allowed = false;
+                       set<string> pass_thru_cmds = context.pass_thru_cmds;
+                       // These commands have special meanings in IPA
+                       context.pass_thru_cmds.insert("!");
+                       context.pass_thru_cmds.insert(";");
+                       context.pass_thru_cmds.insert(":");
                        parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
+                       context.pass_thru_cmds = pass_thru_cmds;
                        context.merging_hyphens_allowed = merging_hyphens_allowed;
                        end_inset(os);
                        preamble.registerAutomaticallyLoadedPackage("tipa");
@@ -4170,21 +4294,37 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        continue;
                }
 
-               // handle refstyle first to catch \eqref which can also occur
-               // without refstyle. Only recognize these commands if
+               // Handle refstyle first in order to to catch \eqref, because this
+               // can also occur without refstyle. Only recognize these commands if
                // refstyle.sty was found in the preamble (otherwise \eqref
                // and user defined ref commands could be misdetected).
-               if ((where = is_known(t.cs(), known_refstyle_commands))
+               // We uncapitalize the input in order to catch capitalized commands
+               // such as \Eqref.
+               if ((where = is_known(uncapitalize(t.cs()), known_refstyle_commands))
                     && preamble.refstyle()) {
+                       string const cap = isCapitalized(t.cs()) ? "true" : "false";
+                       string plural = "false";
+                       // Catch the plural option [s]
+                       if (p.hasOpt()) {
+                               string const opt = p.getOpt();
+                               if (opt == "[s]")
+                                       plural = "true";
+                               else {
+                                       // LyX does not yet support other optional arguments of ref commands
+                                       output_ert_inset(os, t.asInput() + opt + "{" +
+                                              p.verbatim_item() + '}', context);
+                                       continue;
+                               }
+                       }
                        context.check_layout(os);
                        begin_command_inset(os, "ref", "formatted");
                        os << "reference \"";
                        os << known_refstyle_prefixes[where - known_refstyle_commands]
                           << ":";
-                       os << convert_literate_command_inset_arg(p.verbatim_item())
+                       os << convert_literate_command_inset_arg(p.getArg('{', '}'))
                           << "\"\n";
-                       os << "plural \"false\"\n";
-                       os << "caps \"false\"\n";
+                       os << "plural \"" << plural << "\"\n";
+                       os << "caps \"" << cap << "\"\n";
                        os << "noprefix \"false\"\n";
                        end_inset(os);
                        preamble.registerAutomaticallyLoadedPackage("refstyle");
@@ -4213,8 +4353,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        preamble.registerAutomaticallyLoadedPackage("prettyref");
                        } else {
                                // LyX does not yet support optional arguments of ref commands
-                               output_ert_inset(os, t.asInput() + '[' + opt + "]{" +
-                                      p.verbatim_item() + '}', context);
+                               output_ert_inset(os, t.asInput() + opt + "{" +
+                                                p.verbatim_item() + '}', context);
                        }
                        continue;
                }
@@ -4378,13 +4518,13 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        }
                        string keys, pretextlist, posttextlist;
                        if (qualified) {
-                               map<string, string> pres, posts, preslit, postslit;
+                               vector<pair<string, string>> pres, posts, preslit, postslit;
                                vector<string> lkeys;
                                // text before the citation
                                string lbefore, lbeforelit;
                                // text after the citation
                                string lafter, lafterlit;
-                               string lkey;    
+                               string lkey;
                                pair<bool, string> laft, lbef;
                                while (true) {
                                        get_cite_arguments(p, true, lbefore, lafter);
@@ -4395,7 +4535,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                laft = convert_latexed_command_inset_arg(lafter);
                                                literal |= !laft.first;
                                                lafter = laft.second;
-                                               lafterlit = subst(lbefore, "\n", " ");
+                                               lafterlit = subst(lafter, "\n", " ");
                                        }
                                        if (!lbefore.empty()) {
                                                lbefore.erase(0, 1);
@@ -4420,14 +4560,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        lkey = p.getArg('{', '}');
                                        if (lkey.empty())
                                                break;
-                                       if (!lbefore.empty()) {
-                                               pres.insert(make_pair(lkey, lbefore));
-                                               preslit.insert(make_pair(lkey, lbeforelit));
-                                       }
-                                       if (!lafter.empty()) {
-                                               posts.insert(make_pair(lkey, lafter));
-                                               postslit.insert(make_pair(lkey, lafterlit));
-                                       }
+                                       pres.push_back(make_pair(lkey, lbefore));
+                                       preslit.push_back(make_pair(lkey, lbeforelit));
+                                       posts.push_back(make_pair(lkey, lafter));
+                                       postslit.push_back(make_pair(lkey, lafterlit));
                                        lkeys.push_back(lkey);
                                }
                                keys = convert_literate_command_inset_arg(getStringFromVector(lkeys));
@@ -4438,12 +4574,16 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                for (auto const & ptl : pres) {
                                        if (!pretextlist.empty())
                                                pretextlist += '\t';
-                                       pretextlist += ptl.first + " " + ptl.second;
+                                       pretextlist += ptl.first;
+                                       if (!ptl.second.empty())
+                                               pretextlist += " " + ptl.second;
                                }
                                for (auto const & potl : posts) {
                                        if (!posttextlist.empty())
                                                posttextlist += '\t';
-                                       posttextlist += potl.first + " " + potl.second;
+                                       posttextlist += potl.first;
+                                       if (!potl.second.empty())
+                                               posttextlist += " " + potl.second;
                                }
                        } else
                                keys = convert_literate_command_inset_arg(p.verbatim_item());
@@ -4965,6 +5105,13 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        continue;
                }
 
+               if (t.cs() == "endgraf" && context.in_table_cell) {
+                       context.new_paragraph(os);
+                       context.check_layout(os);
+                       skip_spaces_braces(p);
+                       continue;
+               }
+
                if (t.cs() == "input" || t.cs() == "include"
                    || t.cs() == "verbatiminput"
                    || t.cs() == "lstinputlisting"
@@ -5290,6 +5437,14 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                } else
                                        os << "encoding " << '"' << enc->name() << '"' << "\n";
                        }
+                       string bibfileencs;
+                       for (auto const & bf : preamble.biblatex_encodings) {
+                               if (!bibfileencs.empty())
+                                       bibfileencs += "\t";
+                               bibfileencs += bf;
+                       }
+                       if (!bibfileencs.empty())
+                               os << "file_encodings " << '"' << bibfileencs << '"' << "\n";
                        end_inset(os);
                        need_commentbib = false;
                        continue;
@@ -5381,28 +5536,36 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                // the code for the alignment was put here
                // put them in their own if if this is fixed
                if (t.cs() == "fboxrule" || t.cs() == "fboxsep"
-                   || t.cs() == "shadowsize"
-                   || t.cs() == "raggedleft" || t.cs() == "centering"
-                   || t.cs() == "raggedright") {
+                   || t.cs() == "shadowsize") {
                        if (t.cs() == "fboxrule")
                                fboxrule = "";
                        if (t.cs() == "fboxsep")
                                fboxsep = "";
                        if (t.cs() == "shadowsize")
                                shadow_size = "";
-                       if (t.cs() != "raggedleft" && t.cs() != "centering"
-                        && t.cs() != "raggedright") {
+                       p.skip_spaces(true);
+                       while (p.good() && p.next_token().cat() != catSpace
+                              && p.next_token().cat() != catNewline
+                              && p.next_token().cat() != catEscape) {
+                               if (t.cs() == "fboxrule")
+                                       fboxrule = fboxrule + p.get_token().asInput();
+                               if (t.cs() == "fboxsep")
+                                       fboxsep = fboxsep + p.get_token().asInput();
+                               if (t.cs() == "shadowsize")
+                                       shadow_size = shadow_size + p.get_token().asInput();
+                       }
+                       continue;
+               }
+
+               if (t.cs() == "raggedleft" || t.cs() == "centering" || t.cs() == "raggedright") {
+                       if (context.in_table_cell) {
+                               if (t.cs() == "raggedleft")
+                                       context.cell_align = 'r';
+                               else if (t.cs() == "centering")
+                                       context.cell_align = 'c';
+                               else if (t.cs() == "raggedright")
+                                       context.cell_align = 'l';
                                p.skip_spaces(true);
-                               while (p.good() && p.next_token().cat() != catSpace
-                                      && p.next_token().cat() != catNewline
-                                      && p.next_token().cat() != catEscape) {
-                                       if (t.cs() == "fboxrule")
-                                               fboxrule = fboxrule + p.get_token().asInput();
-                                       if (t.cs() == "fboxsep")
-                                               fboxsep = fboxsep + p.get_token().asInput();
-                                       if (t.cs() == "shadowsize")
-                                               shadow_size = shadow_size + p.get_token().asInput();
-                               }
                        } else {
                                output_ert_inset(os, t.asInput(), context);
                        }
@@ -5462,16 +5625,18 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        continue;
                }
 
-               if ((where = is_known(t.cs(), known_spaces))) {
+               if ((where = is_known(t.cs(), known_spaces))
+                   && (context.pass_thru_cmds.find(t.cs()) == context.pass_thru_cmds.end())) {
                        context.check_layout(os);
                        begin_inset(os, "space ");
                        os << '\\' << known_coded_spaces[where - known_spaces]
                           << '\n';
                        end_inset(os);
                        // LaTeX swallows whitespace after all spaces except
-                       // "\\,". We have to do that here, too, because LyX
+                       // "\\,", "\\>", "\\!", "\\;", and "\\:".
+                       // We have to do that here, too, because LyX
                        // adds "{}" which would make the spaces significant.
-                       if (t.cs() !=  ",")
+                       if (!contains(",>!;:", t.cs()))
                                eat_whitespace(p, os, context, false);
                        // LyX adds "{}" after all spaces except "\\ " and
                        // "\\,", so we have to remove "{}".
@@ -5485,7 +5650,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                if (t.cs() == "newpage" ||
                    (t.cs() == "pagebreak" && !p.hasOpt()) ||
                    t.cs() == "clearpage" ||
-                   t.cs() == "cleardoublepage") {
+                   t.cs() == "cleardoublepage" ||
+                   t.cs() == "nopagebreak") {
                        context.check_layout(os);
                        begin_inset(os, "Newpage ");
                        os << t.cs();
@@ -5601,12 +5767,19 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                } else if (unit == "\\bigskipamount") {
                                                        unit = "bigskip";
                                                        known_vspace = true;
+                                               } else if (length == "\\baselineskip") {
+                                                       unit = "fullline";
+                                                       known_vspace = true;
                                                } else if (unit == "\\fill") {
                                                        unit = "vfill";
                                                        known_vspace = true;
                                                }
                                        }
                                }
+                               if (value == 0.5 && t.cs()[0] != 'h' && unit == "\\baselineskip") {
+                                       unit = "halfline";
+                                       known_vspace = true;
+                               }
                                if (!known_hspace && !known_vspace) {
                                        switch (unitFromString(unit)) {
                                        case Length::SP:
@@ -6046,8 +6219,18 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                p.get_token();  // Eat '*'
                                name += '*';
                        }
-                       if (!parse_command(name, p, os, outer, context))
+                       if (!parse_command(name, p, os, outer, context)) {
                                output_ert_inset(os, name, context);
+                               // Try to handle options of unknown commands:
+                               // Look if we have optional arguments,
+                               // and if so, put the brackets in ERT.
+                               while (p.hasOpt()) {
+                                       p.get_token(); // eat '['
+                                       output_ert_inset(os, "[", context);
+                                       os << parse_text_snippet(p, FLAG_BRACK_LAST, outer, context);
+                                       output_ert_inset(os, "]", context);
+                               }
+                       }
                }
        }
 }
@@ -6145,6 +6328,14 @@ void check_comment_bib(ostream & os, Context & context)
        }
        if (!bibfiles.empty())
                os << "bibfiles " << '"' << bibfiles << '"' << "\n";
+       string bibfileencs;
+       for (auto const & bf : preamble.biblatex_encodings) {
+               if (!bibfileencs.empty())
+                       bibfileencs += "\t";
+               bibfileencs += bf;
+       }
+       if (!bibfileencs.empty())
+               os << "file_encodings " << '"' << bibfileencs << '"' << "\n";
        end_inset(os);// Bibtex
        os << "\\end_layout\n";
        end_inset(os);// Note