]> git.lyx.org Git - lyx.git/blobdiff - src/tex2lyx/text.cpp
Cmake build: tex2lyx
[lyx.git] / src / tex2lyx / text.cpp
index 97e4919a8674f6b0ad0ee79cb275264f4963fde9..f406869153eddb30f1ca4fdf8bab2a04ecc4dfb4 100644 (file)
@@ -24,6 +24,8 @@
 #include "Length.h"
 #include "Preamble.h"
 
+#include "insets/ExternalTemplate.h"
+
 #include "support/lassert.h"
 #include "support/convert.h"
 #include "support/FileName.h"
@@ -115,6 +117,22 @@ char const * const known_ref_commands[] = { "ref", "pageref", "vref",
 char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref",
  "vpageref", "formatted", "eqref", 0 };
 
+/**
+ * supported CJK encodings
+ * SJIS and Bg5 cannot be supported as they are not
+ * supported by iconv
+ * JIS does not work with LyX's encoding conversion
+ */
+const char * const supported_CJK_encodings[] = {
+"EUC-JP", "KS", "GB", "UTF8", 0};
+
+/**
+ * the same as supported_CJK_encodings with their corresponding LyX language name
+ * please keep this in sync with supported_CJK_encodings line by line!
+ */
+const char * const supported_CJK_languages[] = {
+"japanese-cjk", "korean", "chinese-simplified", "chinese-traditional", 0};
+
 /*!
  * natbib commands.
  * The starred forms are also known except for "citefullauthor",
@@ -454,11 +472,14 @@ docstring convert_unicodesymbols(docstring s)
                        continue;
                }
                s = s.substr(i);
+               bool termination;
                docstring rem;
                set<string> req;
                docstring parsed = encodings.fromLaTeXCommand(s,
-                               Encodings::TEXT_CMD, rem, &req);
-               for (set<string>::const_iterator it = req.begin(); it != req.end(); it++)
+                               Encodings::TEXT_CMD, termination, rem, &req);
+               set<string>::const_iterator it = req.begin();
+               set<string>::const_iterator en = req.end();
+               for (; it != en; ++it)
                        preamble.registerAutomaticallyLoadedPackage(*it);
                os << parsed;
                s = rem;
@@ -567,7 +588,7 @@ void skip_spaces_braces(Parser & p, bool keepws = false)
           should be handled by this function:
           - abc \j{} xyz
           - abc \j {} xyz
-          - abc \j 
+          - abc \j
             {} xyz
           - abc \j %comment
             {} xyz
@@ -607,27 +628,40 @@ void output_command_layout(ostream & os, Parser & p, bool outer,
        }
        context.check_deeper(os);
        context.check_layout(os);
-       unsigned int optargs = 0;
-       while (optargs < context.layout->optargs) {
+       // FIXME: Adjust to format 446!
+       // Since format 446, layouts do not require anymore all optional
+       // arguments before the required ones. Needs to be implemented!
+       int optargs = 0;
+       while (optargs < context.layout->optArgs()) {
                eat_whitespace(p, os, context, false);
                if (p.next_token().cat() == catEscape ||
-                   p.next_token().character() != '[') 
+                   p.next_token().character() != '[')
                        break;
                p.get_token(); // eat '['
-               begin_inset(os, "Argument\n");
+               // FIXME: Just a workaround. InsetArgument::updateBuffer
+               //        will compute a proper ID for all "999" Arguments
+               //        (which is also what lyx2lyx produces).
+               //        However, tex2lyx should be able to output proper IDs
+               //        itself.
+               begin_inset(os, "Argument 999\n");
                os << "status collapsed\n\n";
                parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
                end_inset(os);
                eat_whitespace(p, os, context, false);
                ++optargs;
        }
-       unsigned int reqargs = 0;
-       while (reqargs < context.layout->reqargs) {
+       int reqargs = 0;
+       while (reqargs < context.layout->requiredArgs()) {
                eat_whitespace(p, os, context, false);
                if (p.next_token().cat() != catBegin)
                        break;
                p.get_token(); // eat '{'
-               begin_inset(os, "Argument\n");
+               // FIXME: Just a workaround. InsetArgument::updateBuffer
+               //        will compute a proper ID for all "999" Arguments
+               //        (which is also what lyx2lyx produces).
+               //        However, tex2lyx should be able to output proper IDs
+               //        itself.
+               begin_inset(os, "Argument 999\n");
                os << "status collapsed\n\n";
                parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
                end_inset(os);
@@ -793,8 +827,18 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
                        if (inner_type != "makebox") {
                                latex_height = p.getArg('[', ']');
                                translate_box_len(latex_height, height_value, height_unit, height_special);
-                       } else
-                               hor_pos = p.getArg('[', ']');
+                       } else {
+                               string const opt = p.getArg('[', ']');
+                               if (!opt.empty()) {
+                                       hor_pos = opt;
+                                       if (hor_pos != "l" && hor_pos != "c" &&
+                                           hor_pos != "r" && hor_pos != "s") {
+                                               cerr << "invalid hor_pos " << hor_pos
+                                                    << " for " << inner_type << endl;
+                                               hor_pos = "c";
+                                       }
+                               }
+                       }
 
                        if (p.hasOpt()) {
                                inner_pos = p.getArg('[', ']');
@@ -818,7 +862,7 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
                        if (!opt.empty()) {
                                hor_pos = opt;
                                if (hor_pos != "l" && hor_pos != "c" &&
-                                   hor_pos != "r") {
+                                   hor_pos != "r" && hor_pos != "s") {
                                        cerr << "invalid hor_pos " << hor_pos
                                             << " for " << outer_type << endl;
                                        hor_pos = "c";
@@ -928,7 +972,7 @@ void parse_box(Parser & p, ostream & os, unsigned outer_flags,
                        // the inner env
                        if (!inner_type.empty() && (inner_flags & FLAG_END))
                                active_environments.pop_back();
-       
+
                        // Ensure that the end of the outer box is parsed correctly:
                        // The opening brace has been eaten by parse_outer_box()
                        if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
@@ -1116,16 +1160,30 @@ void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer,
 }
 
 
-void parse_listings(Parser & p, ostream & os, Context & parent_context)
+void parse_listings(Parser & p, ostream & os, Context & parent_context, bool in_line)
 {
        parent_context.check_layout(os);
        begin_inset(os, "listings\n");
-       os << "inline false\n"
-          << "status collapsed\n";
+       if (p.hasOpt()) {
+               string arg = p.verbatimOption();
+               os << "lstparams " << '"' << arg << '"' << '\n';
+               if (arg.find("\\color") != string::npos)
+                       preamble.registerAutomaticallyLoadedPackage("color");
+       }
+       if (in_line)
+               os << "inline true\n";
+       else
+               os << "inline false\n";
+       os << "status collapsed\n";
        Context context(true, parent_context.textclass);
        context.layout = &parent_context.textclass.plainLayout();
-       context.check_layout(os);
-       string const s = p.verbatimEnvironment("lstlisting");
+       string s;
+       if (in_line) {
+               s = p.plainCommand('!', '!', "lstinline");
+               context.new_paragraph(os);
+               context.check_layout(os);
+       } else
+               s = p.plainEnvironment("lstlisting");
        for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
                if (*it == '\\')
                        os << "\n\\backslash\n";
@@ -1171,8 +1229,7 @@ void parse_unknown_environment(Parser & p, string const & name, ostream & os,
 
 
 void parse_environment(Parser & p, ostream & os, bool outer,
-                       string & last_env, bool & title_layout_found,
-                       Context & parent_context)
+                       string & last_env, Context & parent_context)
 {
        Layout const * newlayout;
        InsetLayout const * newinsetlayout = 0;
@@ -1196,6 +1253,23 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                }
        }
 
+       else if (is_known(name, preamble.polyglossia_languages)) {
+               // We must begin a new paragraph if not already done
+               if (! parent_context.atParagraphStart()) {
+                       parent_context.check_end_layout(os);
+                       parent_context.new_paragraph(os);
+               }
+               // save the language in the context so that it is
+               // handled by parse_text
+               parent_context.font.language = preamble.polyglossia2lyx(name);
+               parse_text(p, os, FLAG_END, outer, parent_context);
+               // Just in case the environment is empty
+               parent_context.extra_stuff.erase();
+               // We must begin a new paragraph to reset the language
+               parent_context.new_paragraph(os);
+               p.skip_spaces();
+       }
+
        else if (unstarred_name == "tabular" || name == "longtable") {
                eat_whitespace(p, os, parent_context, false);
                string width = "0pt";
@@ -1265,7 +1339,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                // we must make sure that the next item gets a \begin_layout.
                parent_context.new_paragraph(os);
                p.skip_spaces();
-                preamble.registerAutomaticallyLoadedPackage("rotfloat");
+               preamble.registerAutomaticallyLoadedPackage("rotfloat");
        }
 
        else if (name == "wrapfigure" || name == "wraptable") {
@@ -1298,7 +1372,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                // we must make sure that the next item gets a \begin_layout.
                parent_context.new_paragraph(os);
                p.skip_spaces();
-                preamble.registerAutomaticallyLoadedPackage("wrapfig");
+               preamble.registerAutomaticallyLoadedPackage("wrapfig");
        }
 
        else if (name == "minipage") {
@@ -1341,6 +1415,81 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                preamble.registerAutomaticallyLoadedPackage("verbatim");
        }
 
+       else if (name == "verbatim") {
+               os << "\n\\end_layout\n\n\\begin_layout Verbatim\n";
+               string const s = p.plainEnvironment("verbatim");
+               string::const_iterator it2 = s.begin();
+               for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
+                       if (*it == '\\')
+                               os << "\\backslash ";
+                       else if (*it == '\n') {
+                               it2 = it + 1;
+                               // avoid adding an empty paragraph at the end
+                               // FIXME: if there are 2 consecutive spaces at the end ignore it
+                               // because LyX will re-add a \n
+                               // This hack must be removed once bug 8049 is fixed!
+                               if ((it + 1 != et) && (it + 2 != et || *it2 != '\n'))
+                                       os << "\n\\end_layout\n\\begin_layout Verbatim\n";
+                       } else
+                               os << *it;
+               }
+               os << "\n\\end_layout\n\n";
+               p.skip_spaces();
+               // reset to Standard layout
+               os << "\n\\begin_layout Standard\n";
+       }
+
+       else if (name == "CJK") {
+               // the scheme is \begin{CJK}{encoding}{mapping}{text}
+               // It is impossible to decide if a CJK environment was in its own paragraph or within
+               // a line. We therefore always assume a paragraph since the latter is a rare case.
+               eat_whitespace(p, os, parent_context, false);
+               parent_context.check_end_layout(os);
+               // store the encoding to be able to reset it
+               string const encoding_old = p.getEncoding();
+               string const encoding = p.getArg('{', '}');
+               // SJIS and Bg5 cannot be handled by iconv
+               // JIS does not work with LyX's encoding conversion
+               if (encoding != "Bg5" && encoding != "JIS" && encoding != "SJIS")
+                       p.setEncoding(encoding);
+               else
+                       p.setEncoding("utf8");
+               // LyX doesn't support the second argument so if
+               // this is used we need to output everything as ERT
+               string const mapping = p.getArg('{', '}');
+               char const * const * const where =
+                       is_known(encoding, supported_CJK_encodings);
+               if ((!mapping.empty() && mapping != " ") || !where) {
+                       parent_context.check_layout(os);
+                       handle_ert(os, "\\begin{" + name + "}{" + encoding + "}{" + mapping + "}",
+                                      parent_context);
+                       // we must parse the content as verbatim because e.g. JIS can contain
+                       // normally invalid characters
+                       string const s = p.plainEnvironment("CJK");
+                       for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
+                               if (*it == '\\')
+                                       handle_ert(os, "\\", parent_context);
+                               else if (*it == '$')
+                                       handle_ert(os, "$", parent_context);
+                               else
+                                       os << *it;
+                       }
+                       handle_ert(os, "\\end{" + name + "}",
+                                      parent_context);
+               } else {
+                       string const lang =
+                               supported_CJK_languages[where - supported_CJK_encodings];
+                       // store the language because we must reset it at the end
+                       string const lang_old = parent_context.font.language;
+                       parent_context.font.language = lang;
+                       parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
+                       parent_context.font.language = lang_old;
+                       parent_context.new_paragraph(os);
+               }
+               p.setEncoding(encoding_old);
+               p.skip_spaces();
+       }
+
        else if (name == "lyxgreyedout") {
                eat_whitespace(p, os, parent_context, false);
                parent_context.check_layout(os);
@@ -1361,14 +1510,7 @@ void parse_environment(Parser & p, ostream & os, bool outer,
 
        else if (name == "lstlisting") {
                eat_whitespace(p, os, parent_context, false);
-               // FIXME handle listings with parameters
-               //       If this is added, don't forgot to handle the
-               //       automatic color package loading
-               if (p.hasOpt())
-                       parse_unknown_environment(p, name, os, FLAG_END,
-                                                 outer, parent_context);
-               else
-                       parse_listings(p, os, parent_context);
+               parse_listings(p, os, parent_context, false);
                p.skip_spaces();
        }
 
@@ -1468,32 +1610,38 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                }
                context.check_deeper(os);
                // handle known optional and required arguments
-               // layouts require all optional arguments before the required ones
+               // FIXME: Since format 446, layouts do not require anymore all optional
+               // arguments before the required ones. Needs to be implemented!
                // Unfortunately LyX can't handle arguments of list arguments (bug 7468):
                // It is impossible to place anything after the environment name,
                // but before the first \\item.
                if (context.layout->latextype == LATEX_ENVIRONMENT) {
                        bool need_layout = true;
-                       unsigned int optargs = 0;
-                       while (optargs < context.layout->optargs) {
+                       int optargs = 0;
+                       while (optargs < context.layout->optArgs()) {
                                eat_whitespace(p, os, context, false);
                                if (p.next_token().cat() == catEscape ||
-                                   p.next_token().character() != '[') 
+                                   p.next_token().character() != '[')
                                        break;
                                p.get_token(); // eat '['
                                if (need_layout) {
                                        context.check_layout(os);
                                        need_layout = false;
                                }
-                               begin_inset(os, "Argument\n");
+                               // FIXME: Just a workaround. InsetArgument::updateBuffer
+                               //        will compute a proper ID for all "999" Arguments
+                               //        (which is also what lyx2lyx produces).
+                               //        However, tex2lyx should be able to output proper IDs
+                               //        itself.
+                               begin_inset(os, "Argument 999\n");
                                os << "status collapsed\n\n";
                                parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
                                end_inset(os);
                                eat_whitespace(p, os, context, false);
                                ++optargs;
                        }
-                       unsigned int reqargs = 0;
-                       while (reqargs < context.layout->reqargs) {
+                       int reqargs = 0;
+                       while (reqargs < context.layout->requiredArgs()) {
                                eat_whitespace(p, os, context, false);
                                if (p.next_token().cat() != catBegin)
                                        break;
@@ -1502,7 +1650,12 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                                        context.check_layout(os);
                                        need_layout = false;
                                }
-                               begin_inset(os, "Argument\n");
+                               // FIXME: Just a workaround. InsetArgument::updateBuffer
+                               //        will compute a proper ID for all "999" Arguments
+                               //        (which is also what lyx2lyx produces).
+                               //        However, tex2lyx should be able to output proper IDs
+                               //        itself.
+                               begin_inset(os, "Argument 999\n");
                                os << "status collapsed\n\n";
                                parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
                                end_inset(os);
@@ -1520,10 +1673,12 @@ void parse_environment(Parser & p, ostream & os, bool outer,
                context.check_end_deeper(os);
                parent_context.new_paragraph(os);
                p.skip_spaces();
-               if (!title_layout_found)
-                       title_layout_found = newlayout->intitle;
+               if (!preamble.titleLayoutFound())
+                       preamble.titleLayoutFound(newlayout->intitle);
                set<string> const & req = newlayout->requires();
-               for (set<string>::const_iterator it = req.begin(); it != req.end(); it++)
+               set<string>::const_iterator it = req.begin();
+               set<string>::const_iterator en = req.end();
+               for (; it != en; ++it)
                        preamble.registerAutomaticallyLoadedPackage(*it);
        }
 
@@ -1707,37 +1862,109 @@ string const normalize_filename(string const & name)
 
 /// Convert \p name from TeX convention (relative to master file) to LyX
 /// convention (relative to .lyx file) if it is relative
-void fix_relative_filename(string & name)
+void fix_child_filename(string & name)
 {
-       if (FileName::isAbsolute(name))
-               return;
+       string const absMasterTeX = getMasterFilePath(true);
+       bool const isabs = FileName::isAbsolute(name);
+       // convert from "relative to .tex master" to absolute original path
+       if (!isabs)
+               name = makeAbsPath(name, absMasterTeX).absFileName();
+       bool copyfile = copyFiles();
+       string const absParentLyX = getParentFilePath(false);
+       string abs = name;
+       if (copyfile) {
+               // convert from absolute original path to "relative to master file"
+               string const rel = to_utf8(makeRelPath(from_utf8(name),
+                                                      from_utf8(absMasterTeX)));
+               // re-interpret "relative to .tex file" as "relative to .lyx file"
+               // (is different if the master .lyx file resides in a
+               // different path than the master .tex file)
+               string const absMasterLyX = getMasterFilePath(false);
+               abs = makeAbsPath(rel, absMasterLyX).absFileName();
+               // Do not copy if the new path is impossible to create. Example:
+               // absMasterTeX = "/foo/bar/"
+               // absMasterLyX = "/bar/"
+               // name = "/baz.eps" => new absolute name would be "/../baz.eps"
+               if (contains(name, "/../"))
+                       copyfile = false;
+       }
+       if (copyfile) {
+               if (isabs)
+                       name = abs;
+               else {
+                       // convert from absolute original path to
+                       // "relative to .lyx file"
+                       name = to_utf8(makeRelPath(from_utf8(abs),
+                                                  from_utf8(absParentLyX)));
+               }
+       }
+       else if (!isabs) {
+               // convert from absolute original path to "relative to .lyx file"
+               name = to_utf8(makeRelPath(from_utf8(name),
+                                          from_utf8(absParentLyX)));
+       }
+}
 
-       name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFileName()),
-                                  from_utf8(getParentFilePath())));
+
+void copy_file(FileName const & src, string dstname)
+{
+       if (!copyFiles())
+               return;
+       string const absParent = getParentFilePath(false);
+       FileName dst;
+       if (FileName::isAbsolute(dstname))
+               dst = FileName(dstname);
+       else
+               dst = makeAbsPath(dstname, absParent);
+       string const absMaster = getMasterFilePath(false);
+       FileName const srcpath = src.onlyPath();
+       FileName const dstpath = dst.onlyPath();
+       if (equivalent(srcpath, dstpath))
+               return;
+       if (!dstpath.isDirectory()) {
+               if (!dstpath.createPath()) {
+                       cerr << "Warning: Could not create directory for file `"
+                            << dst.absFileName() << "´." << endl;
+                       return;
+               }
+       }
+       if (dst.isReadableFile()) {
+               if (overwriteFiles())
+                       cerr << "Warning: Overwriting existing file `"
+                            << dst.absFileName() << "´." << endl;
+               else {
+                       cerr << "Warning: Not overwriting existing file `"
+                            << dst.absFileName() << "´." << endl;
+                       return;
+               }
+       }
+       if (!src.copyTo(dst))
+               cerr << "Warning: Could not copy file `" << src.absFileName()
+                    << "´ to `" << dst.absFileName() << "´." << endl;
 }
 
 
-/// Parse a NoWeb Scrap section. The initial "<<" is already parsed.
+/// Parse a NoWeb Chunk section. The initial "<<" is already parsed.
 void parse_noweb(Parser & p, ostream & os, Context & context)
 {
        // assemble the rest of the keyword
        string name("<<");
-       bool scrap = false;
+       bool chunk = false;
        while (p.good()) {
                Token const & t = p.get_token();
                if (t.asInput() == ">" && p.next_token().asInput() == ">") {
                        name += ">>";
                        p.get_token();
-                       scrap = (p.good() && p.next_token().asInput() == "=");
-                       if (scrap)
+                       chunk = (p.good() && p.next_token().asInput() == "=");
+                       if (chunk)
                                name += p.get_token().asInput();
                        break;
                }
                name += t.asInput();
        }
 
-       if (!scrap || !context.new_layout_allowed ||
-           !context.textclass.hasLayout(from_ascii("Scrap"))) {
+       if (!chunk || !context.new_layout_allowed ||
+           !context.textclass.hasLayout(from_ascii("Chunk"))) {
                cerr << "Warning: Could not interpret '" << name
                     << "'. Ignoring it." << endl;
                return;
@@ -1751,7 +1978,7 @@ void parse_noweb(Parser & p, ostream & os, Context & context)
        // always must be in an own paragraph.
        context.new_paragraph(os);
        Context newcontext(true, context.textclass,
-               &context.textclass[from_ascii("Scrap")]);
+               &context.textclass[from_ascii("Chunk")]);
        newcontext.check_layout(os);
        os << name;
        while (p.good()) {
@@ -1763,12 +1990,12 @@ void parse_noweb(Parser & p, ostream & os, Context & context)
                else {
                        ostringstream oss;
                        Context tmp(false, context.textclass,
-                                   &context.textclass[from_ascii("Scrap")]);
+                                   &context.textclass[from_ascii("Chunk")]);
                        tmp.need_end_layout = true;
                        tmp.check_layout(oss);
                        os << subst(t.asInput(), "\n", oss.str());
                }
-               // The scrap chunk is ended by an @ at the beginning of a line.
+               // The chunk is ended by an @ at the beginning of a line.
                // After the @ the line may contain a comment and/or
                // whitespace, but nothing else.
                if (t.asInput() == "@" && p.prev_token().cat() == catNewline &&
@@ -1899,6 +2126,28 @@ void parse_macro(Parser & p, ostream & os, Context & context)
                handle_ert(os, command + ert, context);
 }
 
+
+void registerExternalTemplatePackages(string const & name)
+{
+       external::TemplateManager const & etm = external::TemplateManager::get();
+       external::Template const * const et = etm.getTemplateByName(name);
+       if (!et)
+               return;
+       external::Template::Formats::const_iterator cit = et->formats.end();
+       if (pdflatex)
+               cit = et->formats.find("PDFLaTeX");
+       if (cit == et->formats.end())
+               // If the template has not specified a PDFLaTeX output,
+               // we try the LaTeX format.
+               cit = et->formats.find("LaTeX");
+       if (cit == et->formats.end())
+               return;
+       vector<string>::const_iterator qit = cit->second.requirements.begin();
+       vector<string>::const_iterator qend = cit->second.requirements.end();
+       for (; qit != qend; ++qit)
+               preamble.registerAutomaticallyLoadedPackage(*qit);
+}
+
 } // anonymous namespace
 
 
@@ -1911,14 +2160,32 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
        // Store the latest bibliographystyle and nocite{*} option
        // (needed for bibtex inset)
        string btprint;
-       string bibliographystyle;
+       string bibliographystyle = "default";
        bool const use_natbib = preamble.isPackageUsed("natbib");
        bool const use_jurabib = preamble.isPackageUsed("jurabib");
        string last_env;
-       bool title_layout_found = false;
        while (p.good()) {
                Token const & t = p.get_token();
 
+       // it is impossible to determine the correct encoding for non-CJK Japanese.
+       // Therefore write a note at the beginning of the document
+       if (is_nonCJKJapanese) {
+               context.check_layout(os);
+               begin_inset(os, "Note Note\n");
+               os << "status open\n\\begin_layout Plain Layout\n"
+                  << "\\series bold\n"
+                  << "Important information:\n"
+                  << "\\end_layout\n\n"
+                  << "\\begin_layout Plain Layout\n"
+                  << "This document is in Japanese (non-CJK).\n"
+                  << " It was therefore impossible for tex2lyx to determine the correct encoding."
+                  << " The encoding EUC-JP was assumed. If this is incorrect, please set the correct"
+                  << " encoding in the document settings.\n"
+                  << "\\end_layout\n";
+               end_inset(os);
+               is_nonCJKJapanese = false;
+       }
+
 #ifdef FILEDEBUG
                debugToken(cerr, t, flags);
 #endif
@@ -2015,6 +2282,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        skip_braces(p);
                }
 
+               else if (t.asInput() == "<"
+                        && p.next_token().asInput() == "<" && noweb_mode) {
+                       p.get_token();
+                       parse_noweb(p, os, context);
+               }
+
                else if (t.asInput() == "<" && p.next_token().asInput() == "<") {
                        context.check_layout(os);
                        begin_inset(os, "Quotes ");
@@ -2024,12 +2297,6 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        skip_braces(p);
                }
 
-               else if (t.asInput() == "<"
-                        && p.next_token().asInput() == "<" && noweb_mode) {
-                       p.get_token();
-                       parse_noweb(p, os, context);
-               }
-
                else if (t.cat() == catSpace || (t.cat() == catNewline && ! p.isParagraph()))
                        check_space(p, os, context);
 
@@ -2242,7 +2509,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                else if (t.cs() == "begin")
                        parse_environment(p, os, outer, last_env,
-                                         title_layout_found, context);
+                                         context);
 
                else if (t.cs() == "end") {
                        if (flags & FLAG_END) {
@@ -2346,7 +2613,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                skip_braces(p);
                                                p.get_token();
                                                string name = normalize_filename(p.verbatim_item());
-                                               string const path = getMasterFilePath();
+                                               string const path = getMasterFilePath(true);
                                                // We want to preserve relative / absolute filenames,
                                                // therefore path is only used for testing
                                                // The file extension is in every case ".tex".
@@ -2361,9 +2628,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                        if (!Gnumeric_name.empty())
                                                                name = Gnumeric_name;
                                                }
-                                               if (makeAbsPath(name, path).exists())
-                                                       fix_relative_filename(name);
-                                               else
+                                               FileName const absname = makeAbsPath(name, path);
+                                               if (absname.exists()) {
+                                                       fix_child_filename(name);
+                                                       copy_file(absname, name);
+                                               } else
                                                        cerr << "Warning: Could not find file '"
                                                             << name << "'." << endl;
                                                context.check_layout(os);
@@ -2373,16 +2642,9 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                                end_inset(os);
                                                context.check_layout(os);
                                                macro = false;
-                                               // register the packages that are automatically reloaded
+                                               // register the packages that are automatically loaded
                                                // by the Gnumeric template
-                                               // Fixme: InsetExternal.cpp should give us that list
-                                               preamble.registerAutomaticallyLoadedPackage("array");
-                                               preamble.registerAutomaticallyLoadedPackage("calc");
-                                               preamble.registerAutomaticallyLoadedPackage("color");
-                                               preamble.registerAutomaticallyLoadedPackage("hhline");
-                                               preamble.registerAutomaticallyLoadedPackage("ifthen");
-                                               preamble.registerAutomaticallyLoadedPackage("longtable");
-                                               preamble.registerAutomaticallyLoadedPackage("multirow");
+                                               registerExternalTemplatePackages("GnumericSpreadsheet");
                                        }
                                }
                        }
@@ -2420,10 +2682,14 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                // Must catch empty dates before findLayout is called below
                else if (t.cs() == "date") {
+                       eat_whitespace(p, os, context, false);
+                       p.pushPosition();
                        string const date = p.verbatim_item();
-                       if (date.empty())
+                       p.popPosition();
+                       if (date.empty()) {
                                preamble.suppressDate(true);
-                       else {
+                               p.verbatim_item();
+                       } else {
                                preamble.suppressDate(false);
                                if (context.new_layout_allowed &&
                                    (newlayout = findLayout(context.textclass,
@@ -2431,16 +2697,18 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        // write the layout
                                        output_command_layout(os, p, outer,
                                                        context, newlayout);
-                                       p.skip_spaces();
-                                       if (!title_layout_found)
-                                               title_layout_found = newlayout->intitle;
+                                       parse_text_snippet(p, os, FLAG_ITEM, outer, context);
+                                       if (!preamble.titleLayoutFound())
+                                               preamble.titleLayoutFound(newlayout->intitle);
                                        set<string> const & req = newlayout->requires();
-                                       for (set<string>::const_iterator it = req.begin();
-                                            it != req.end(); it++)
+                                       set<string>::const_iterator it = req.begin();
+                                       set<string>::const_iterator en = req.end();
+                                       for (; it != en; ++it)
                                                preamble.registerAutomaticallyLoadedPackage(*it);
                                } else
-                                       handle_ert(os, "\\date{" + date + '}',
-                                                       context);
+                                       handle_ert(os,
+                                               "\\date{" + p.verbatim_item() + '}',
+                                               context);
                        }
                }
 
@@ -2453,10 +2721,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        p.get_token();
                        output_command_layout(os, p, outer, context, newlayout);
                        p.skip_spaces();
-                       if (!title_layout_found)
-                               title_layout_found = newlayout->intitle;
+                       if (!preamble.titleLayoutFound())
+                               preamble.titleLayoutFound(newlayout->intitle);
                        set<string> const & req = newlayout->requires();
-                       for (set<string>::const_iterator it = req.begin(); it != req.end(); it++)
+                       for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
                                preamble.registerAutomaticallyLoadedPackage(*it);
                }
 
@@ -2466,10 +2734,10 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // write the layout
                        output_command_layout(os, p, outer, context, newlayout);
                        p.skip_spaces();
-                       if (!title_layout_found)
-                               title_layout_found = newlayout->intitle;
+                       if (!preamble.titleLayoutFound())
+                               preamble.titleLayoutFound(newlayout->intitle);
                        set<string> const & req = newlayout->requires();
-                       for (set<string>::const_iterator it = req.begin(); it != req.end(); it++)
+                       for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
                                preamble.registerAutomaticallyLoadedPackage(*it);
                }
 
@@ -2478,13 +2746,14 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        context.check_layout(os);
                        p.skip_spaces();
                        begin_inset(os, "Caption\n");
-                       Context newcontext(true, context.textclass);
-                       newcontext.font = context.font;
+                       Context newcontext(true, context.textclass, 0, 0, context.font);
                        newcontext.check_layout(os);
+                       // FIXME InsetArgument is now properly implemented in InsetLayout
+                       //       (for captions, but also for others)
                        if (p.next_token().cat() != catEscape &&
                            p.next_token().character() == '[') {
                                p.get_token(); // eat '['
-                               begin_inset(os, "Argument\n");
+                               begin_inset(os, "Argument 1\n");
                                os << "status collapsed\n";
                                parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
                                end_inset(os);
@@ -2529,8 +2798,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        os << "\n\\begin_layout Plain Layout";
                                        p.skip_spaces();
                                        begin_inset(os, "Caption\n");
-                                       Context newcontext(true, context.textclass);
-                                       newcontext.font = context.font;
+                                       Context newcontext(true, context.textclass,
+                                                          0, 0, context.font);
                                        newcontext.check_layout(os);
                                        os << caption << "\n";
                                        newcontext.check_end_layout(os);
@@ -2574,7 +2843,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                opts["clip"] = string();
                        string name = normalize_filename(p.verbatim_item());
 
-                       string const path = getMasterFilePath();
+                       string const path = getMasterFilePath(true);
                        // We want to preserve relative / absolute filenames,
                        // therefore path is only used for testing
                        if (!makeAbsPath(name, path).exists()) {
@@ -2609,9 +2878,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                }
                        }
 
-                       if (makeAbsPath(name, path).exists())
-                               fix_relative_filename(name);
-                       else
+                       FileName const absname = makeAbsPath(name, path);
+                       if (absname.exists()) {
+                               fix_child_filename(name);
+                               copy_file(absname, name);
+                       } else
                                cerr << "Warning: Could not find graphics file '"
                                     << name << "'." << endl;
 
@@ -2719,6 +2990,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // Warn about invalid options.
                        // Check whether some option was given twice.
                        end_inset(os);
+                       preamble.registerAutomaticallyLoadedPackage("graphicx");
                }
 
                else if (t.cs() == "footnote" ||
@@ -2740,6 +3012,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        end_inset(os);
                }
 
+               else if (t.cs() == "lstinline") {
+                       p.skip_spaces();
+                       parse_listings(p, os, context, true);
+               }
+
                else if (t.cs() == "ensuremath") {
                        p.skip_spaces();
                        context.check_layout(os);
@@ -2753,18 +3030,20 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                }
 
                else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
-                       if (title_layout_found) {
+                       if (preamble.titleLayoutFound()) {
                                // swallow this
                                skip_spaces_braces(p);
                        } else
                                handle_ert(os, t.asInput(), context);
                }
 
-               else if (t.cs() == "tableofcontents") {
+               else if (t.cs() == "tableofcontents" || t.cs() == "lstlistoflistings") {
                        context.check_layout(os);
-                       begin_command_inset(os, "toc", "tableofcontents");
+                       begin_command_inset(os, "toc", t.cs());
                        end_inset(os);
                        skip_spaces_braces(p);
+                       if (t.cs() == "lstlistoflistings")
+                               preamble.registerAutomaticallyLoadedPackage("listings");
                }
 
                else if (t.cs() == "listoffigures") {
@@ -2939,8 +3218,8 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
 
                else if (t.cs() == "href") {
                        context.check_layout(os);
-                       string target = p.getArg('{', '}');
-                       string name = p.getArg('{', '}');
+                       string target = convert_command_inset_arg(p.verbatim_item());
+                       string name = convert_command_inset_arg(p.verbatim_item());
                        string type;
                        size_t i = target.find(':');
                        if (i != string::npos) {
@@ -2960,7 +3239,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        end_inset(os);
                        skip_spaces_braces(p);
                }
-               
+
                else if (t.cs() == "lyxline") {
                        // swallow size argument (it is not used anyway)
                        p.getArg('{', '}');
@@ -3160,12 +3439,18 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                btprint = key;
                }
 
-               else if (t.cs() == "index") {
+               else if (t.cs() == "index" ||
+                        (t.cs() == "sindex" && preamble.use_indices() == "true")) {
                        context.check_layout(os);
-                       begin_inset(os, "Index idx\n");
-                       os << "status collapsed\n";
+                       string const arg = (t.cs() == "sindex" && p.hasOpt()) ?
+                               p.getArg('[', ']') : "";
+                       string const kind = arg.empty() ? "idx" : arg;
+                       begin_inset(os, "Index ");
+                       os << kind << "\nstatus collapsed\n";
                        parse_text_in_inset(p, os, FLAG_ITEM, false, context, "Index");
                        end_inset(os);
+                       if (kind != "idx")
+                               preamble.registerAutomaticallyLoadedPackage("splitidx");
                }
 
                else if (t.cs() == "nomenclature") {
@@ -3182,7 +3467,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        end_inset(os);
                        preamble.registerAutomaticallyLoadedPackage("nomencl");
                }
-               
+
                else if (t.cs() == "label") {
                        context.check_layout(os);
                        begin_command_inset(os, "label", "label");
@@ -3329,8 +3614,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "selectlanguage") {
                        context.check_layout(os);
                        // save the language for the case that a
-                       // \foreignlanguage is used 
-
+                       // \foreignlanguage is used
                        context.font.language = babel2lyx(p.verbatim_item());
                        os << "\n\\lang " << context.font.language << "\n";
                }
@@ -3342,6 +3626,36 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                              context.font.language, lang);
                }
 
+               else if (prefixIs(t.cs(), "text")
+                        && is_known(t.cs().substr(4), preamble.polyglossia_languages)) {
+                       // scheme is \textLANGUAGE{text} where LANGUAGE is in polyglossia_languages[]
+                       string lang;
+                       // We have to output the whole command if it has an option
+                       // because LyX doesn't support this yet, see bug #8214,
+                       // only if there is a single option specifying a variant, we can handle it.
+                       if (p.hasOpt()) {
+                               string langopts = p.getOpt();
+                               // check if the option contains a variant, if yes, extract it
+                               string::size_type pos_var = langopts.find("variant");
+                               string::size_type i = langopts.find(',');
+                               string::size_type k = langopts.find('=', pos_var);
+                               if (pos_var != string::npos && i == string::npos) {
+                                       string variant;
+                                       variant = langopts.substr(k + 1, langopts.length() - k - 2);
+                                       lang = preamble.polyglossia2lyx(variant);
+                                       parse_text_attributes(p, os, FLAG_ITEM, outer,
+                                                                 context, "\\lang",
+                                                                 context.font.language, lang);
+                               } else
+                                       handle_ert(os, t.asInput() + langopts, context);
+                       } else {
+                               lang = preamble.polyglossia2lyx(t.cs().substr(4, string::npos));
+                               parse_text_attributes(p, os, FLAG_ITEM, outer,
+                                                         context, "\\lang",
+                                                         context.font.language, lang);
+                       }
+               }
+
                else if (t.cs() == "inputencoding") {
                        // nothing to write here
                        string const enc = subst(p.verbatim_item(), "\n", " ");
@@ -3423,7 +3737,11 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                else if (t.cs() == "verb") {
                        context.check_layout(os);
                        char const delimiter = p.next_token().character();
-                       string const arg = p.getArg(delimiter, delimiter);
+                       // \verb is special: The usual escaping rules do not
+                       // apply, e.g. "\verb+\+" is valid and denotes a single
+                       // backslash (bug #4468). Therefore we do not allow
+                       // escaping in getArg().
+                       string const arg = p.getArg(delimiter, delimiter, false);
                        ostringstream oss;
                        oss << "\\verb" << delimiter << arg << delimiter;
                        handle_ert(os, oss.str(), context);
@@ -3436,24 +3754,26 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        handle_ert(os, t.asInput(), context);
 
                // accents (see Table 6 in Comprehensive LaTeX Symbol List)
-               else if (t.cs().size() == 1 
+               else if (t.cs().size() == 1
                         && contains("\"'.=^`bcdHkrtuv~", t.cs())) {
                        context.check_layout(os);
                        // try to see whether the string is in unicodesymbols
+                       bool termination;
                        docstring rem;
-                       string command = t.asInput() + "{" 
+                       string command = t.asInput() + "{"
                                + trimSpaceAndEol(p.verbatim_item())
                                + "}";
                        set<string> req;
                        docstring s = encodings.fromLaTeXCommand(from_utf8(command),
-                               Encodings::TEXT_CMD | Encodings::MATH_CMD, rem, &req);
+                               Encodings::TEXT_CMD | Encodings::MATH_CMD,
+                               termination, rem, &req);
                        if (!s.empty()) {
                                if (!rem.empty())
-                                       cerr << "When parsing " << command 
+                                       cerr << "When parsing " << command
                                             << ", result is " << to_utf8(s)
                                             << "+" << to_utf8(rem) << endl;
                                os << to_utf8(s);
-                               for (set<string>::const_iterator it = req.begin(); it != req.end(); it++)
+                               for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
                                        preamble.registerAutomaticallyLoadedPackage(*it);
                        } else
                                // we did not find a non-ert version
@@ -3494,7 +3814,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                name += p.get_token().asInput();
                        context.check_layout(os);
                        string filename(normalize_filename(p.getArg('{', '}')));
-                       string const path = getMasterFilePath();
+                       string const path = getMasterFilePath(true);
                        // We want to preserve relative / absolute filenames,
                        // therefore path is only used for testing
                        if ((t.cs() == "include" || t.cs() == "input") &&
@@ -3512,16 +3832,17 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        if (makeAbsPath(filename, path).exists()) {
                                string const abstexname =
                                        makeAbsPath(filename, path).absFileName();
-                               string const abslyxname =
-                                       changeExtension(abstexname, ".lyx");
                                string const absfigname =
                                        changeExtension(abstexname, ".fig");
-                               fix_relative_filename(filename);
+                               fix_child_filename(filename);
                                string const lyxname =
                                        changeExtension(filename, ".lyx");
+                               string const abslyxname = makeAbsPath(
+                                       lyxname, getParentFilePath(false)).absFileName();
                                bool xfig = false;
-                               external = FileName(absfigname).exists();
-                               if (t.cs() == "input") {
+                               if (!skipChildren())
+                                       external = FileName(absfigname).exists();
+                               if (t.cs() == "input" && !skipChildren()) {
                                        string const ext = getExtension(abstexname);
 
                                        // Combined PS/LaTeX:
@@ -3564,16 +3885,25 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                }
                                if (external) {
                                        outname = changeExtension(filename, ".fig");
+                                       FileName abssrc(changeExtension(abstexname, ".fig"));
+                                       copy_file(abssrc, outname);
                                } else if (xfig) {
                                        // Don't try to convert, the result
                                        // would be full of ERT.
                                        outname = filename;
+                                       FileName abssrc(abstexname);
+                                       copy_file(abssrc, outname);
                                } else if (t.cs() != "verbatiminput" &&
+                                          !skipChildren() &&
                                    tex2lyx(abstexname, FileName(abslyxname),
                                            p.getEncoding())) {
                                        outname = lyxname;
+                                       // no need to call copy_file
+                                       // tex2lyx creates the file
                                } else {
                                        outname = filename;
+                                       FileName abssrc(abstexname);
+                                       copy_file(abssrc, outname);
                                }
                        } else {
                                cerr << "Warning: Could not find included file '"
@@ -3584,6 +3914,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                begin_inset(os, "External\n");
                                os << "\ttemplate XFig\n"
                                   << "\tfilename " << outname << '\n';
+                               registerExternalTemplatePackages("XFig");
                        } else {
                                begin_command_inset(os, "include", name);
                                os << "preview false\n"
@@ -3681,14 +4012,21 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                arg += p.getFullOpt();
                                eat_whitespace(p, os, context, false);
                                handle_ert(os, arg + '{', context);
-                               eat_whitespace(p, os, context, false);
                                parse_text(p, os, FLAG_ITEM, outer, context);
                                handle_ert(os, "}", context);
                        } else {
                                string special = p.getFullOpt();
                                special += p.getOpt();
-                               parse_outer_box(p, os, FLAG_ITEM, outer,
-                                               context, t.cs(), special);
+                               // LyX does not yet support \framebox without any option
+                               if (!special.empty())
+                                       parse_outer_box(p, os, FLAG_ITEM, outer,
+                                                       context, t.cs(), special);
+                               else {
+                                       eat_whitespace(p, os, context, false);
+                                       handle_ert(os, "\\framebox{", context);
+                                       parse_text(p, os, FLAG_ITEM, outer, context);
+                                       handle_ert(os, "}", context);
+                               }
                        }
                }
 
@@ -3702,7 +4040,6 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                arg += p.getFullOpt();
                                eat_whitespace(p, os, context, false);
                                handle_ert(os, arg + '{', context);
-                               eat_whitespace(p, os, context, false);
                                parse_text(p, os, FLAG_ITEM, outer, context);
                                handle_ert(os, "}", context);
                        } else
@@ -3947,7 +4284,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        vector<string> keys;
                        split_map(arg, opts, keys);
                        string name = normalize_filename(p.verbatim_item());
-                       string const path = getMasterFilePath();
+                       string const path = getMasterFilePath(true);
                        // We want to preserve relative / absolute filenames,
                        // therefore path is only used for testing
                        if (!makeAbsPath(name, path).exists()) {
@@ -3961,9 +4298,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                        pdflatex = true;
                                }
                        }
-                       if (makeAbsPath(name, path).exists())
-                               fix_relative_filename(name);
-                       else
+                       FileName const absname = makeAbsPath(name, path);
+                       if (absname.exists())
+                       {
+                               fix_child_filename(name);
+                               copy_file(absname, name);
+                       } else
                                cerr << "Warning: Could not find file '"
                                     << name << "'." << endl;
                        // write output
@@ -4006,12 +4346,13 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                os << "\tkeepAspectRatio\n";
                        end_inset(os);
                        context.check_layout(os);
+                       registerExternalTemplatePackages("PDFPages");
                }
 
                else if (t.cs() == "loadgame") {
                        p.skip_spaces();
                        string name = normalize_filename(p.verbatim_item());
-                       string const path = getMasterFilePath();
+                       string const path = getMasterFilePath(true);
                        // We want to preserve relative / absolute filenames,
                        // therefore path is only used for testing
                        if (!makeAbsPath(name, path).exists()) {
@@ -4023,9 +4364,12 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                                if (!lyxskak_name.empty())
                                        name = lyxskak_name;
                        }
-                       if (makeAbsPath(name, path).exists())
-                               fix_relative_filename(name);
-                       else
+                       FileName const absname = makeAbsPath(name, path);
+                       if (absname.exists())
+                       {
+                               fix_child_filename(name);
+                               copy_file(absname, name);
+                       } else
                                cerr << "Warning: Could not find file '"
                                     << name << "'." << endl;
                        context.check_layout(os);
@@ -4037,25 +4381,28 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        // after a \loadgame follows a \showboard
                        if (p.get_token().asInput() == "showboard")
                                p.get_token();
+                       registerExternalTemplatePackages("ChessDiagram");
                }
 
                else {
                        // try to see whether the string is in unicodesymbols
                        // Only use text mode commands, since we are in text mode here,
                        // and math commands may be invalid (bug 6797)
+                       bool termination;
                        docstring rem;
                        set<string> req;
                        docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
-                                                                Encodings::TEXT_CMD, rem, &req);
+                                       Encodings::TEXT_CMD, termination, rem, &req);
                        if (!s.empty()) {
                                if (!rem.empty())
-                                       cerr << "When parsing " << t.cs() 
+                                       cerr << "When parsing " << t.cs()
                                             << ", result is " << to_utf8(s)
                                             << "+" << to_utf8(rem) << endl;
                                context.check_layout(os);
                                os << to_utf8(s);
-                               skip_spaces_braces(p);
-                               for (set<string>::const_iterator it = req.begin(); it != req.end(); it++)
+                               if (termination)
+                                       skip_spaces_braces(p);
+                               for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
                                        preamble.registerAutomaticallyLoadedPackage(*it);
                        }
                        //cerr << "#: " << t << " mode: " << mode << endl;
@@ -4063,7 +4410,7 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
                        /*
                        string s = t.asInput();
                        string z = p.verbatim_item();
-                       while (p.good() && z != " " && z.size()) {
+                       while (p.good() && z != " " && !z.empty()) {
                                //cerr << "read: " << z << endl;
                                s += z;
                                z = p.verbatim_item();
@@ -4090,6 +4437,79 @@ void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
        }
 }
 
+
+string guessLanguage(Parser & p, string const & lang)
+{
+       typedef std::map<std::string, size_t> LangMap;
+       // map from language names to number of characters
+       LangMap used;
+       used[lang] = 0;
+       for (char const * const * i = supported_CJK_languages; *i; i++)
+               used[string(*i)] = 0;
+
+       while (p.good()) {
+               Token const t = p.get_token();
+               // comments are not counted for any language
+               if (t.cat() == catComment)
+                       continue;
+               // commands are not counted as well, but we need to detect
+               // \begin{CJK} and switch encoding if needed
+               if (t.cat() == catEscape) {
+                       if (t.cs() == "inputencoding") {
+                               string const enc = subst(p.verbatim_item(), "\n", " ");
+                               p.setEncoding(enc);
+                               continue;
+                       }
+                       if (t.cs() != "begin")
+                               continue;
+               } else {
+                       // Non-CJK content is counted for lang.
+                       // We do not care about the real language here:
+                       // If we have more non-CJK contents than CJK contents,
+                       // we simply use the language that was specified as
+                       // babel main language.
+                       used[lang] += t.asInput().length();
+                       continue;
+               }
+               // Now we are starting an environment
+               p.pushPosition();
+               string const name = p.getArg('{', '}');
+               if (name != "CJK") {
+                       p.popPosition();
+                       continue;
+               }
+               // It is a CJK environment
+               p.popPosition();
+               /* name = */ p.getArg('{', '}');
+               string const encoding = p.getArg('{', '}');
+               /* mapping = */ p.getArg('{', '}');
+               string const encoding_old = p.getEncoding();
+               char const * const * const where =
+                       is_known(encoding, supported_CJK_encodings);
+               if (where)
+                       p.setEncoding(encoding);
+               else
+                       p.setEncoding("utf8");
+               string const text = p.verbatimEnvironment("CJK");
+               p.setEncoding(encoding_old);
+               p.skip_spaces();
+               if (!where) {
+                       // ignore contents in unknown CJK encoding
+                       continue;
+               }
+               // the language of the text
+               string const cjk =
+                       supported_CJK_languages[where - supported_CJK_encodings];
+               used[cjk] += text.length();
+       }
+       LangMap::const_iterator use = used.begin();
+       for (LangMap::const_iterator it = used.begin(); it != used.end(); ++it) {
+               if (it->second > use->second)
+                       use = it;
+       }
+       return use->first;
+}
+
 // }])