]> git.lyx.org Git - lyx.git/blobdiff - src/tex2lyx/Preamble.cpp
BufferParams: rename formula_indent to math_indent etc. as requested
[lyx.git] / src / tex2lyx / Preamble.cpp
index 6ea980eb1db69c5ab1ed0b0d8a1cc735e1638523..08d019c49e6bf30e3448c981314c3f1eee035c9e 100644 (file)
 #include "Preamble.h"
 #include "tex2lyx.h"
 
+#include "Encoding.h"
 #include "LayoutFile.h"
 #include "Layout.h"
 #include "Lexer.h"
 #include "TextClass.h"
+#include "version.h"
 
 #include "support/convert.h"
 #include "support/FileName.h"
@@ -37,16 +39,12 @@ using namespace lyx::support;
 
 namespace lyx {
 
-// special columntypes
-extern map<char, int> special_columns;
-
 Preamble preamble;
 
 namespace {
 
-// "chinese-simplified", "chinese-traditional", "japanese-cjk", "korean"
-// cannot be supported because it is impossible to determine the correct document
-// language if CJK is used.
+// CJK languages are handled in text.cpp, polyglossia languages are listed
+// further down.
 /**
  * known babel language names (including synonyms)
  * not in standard babel: arabic, arabtex, armenian, belarusian, serbian-latin, thai
@@ -54,18 +52,19 @@ namespace {
  */
 const char * const known_languages[] = {"acadian", "afrikaans", "albanian",
 "american", "arabic", "arabtex", "australian", "austrian", "bahasa", "bahasai",
-"bahasam", "basque", "belarusian", "brazil", "brazilian", "breton", "british",
+"bahasam", "basque", "belarusian", "bosnian", "brazil", "brazilian", "breton", "british",
 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "francais",
-"french", "frenchb", "frenchle", "frenchpro", "galician", "german", "germanb",
-"greek", "hebrew", "hungarian", "icelandic", "indon", "indonesian", "interlingua",
-"irish", "italian", "japanese", "kazakh", "kurmanji", "latin", "latvian", "lithuanian",
-"lowersorbian", "lsorbian", "magyar", "malay", "meyalu", "mongolian", "naustrian",
-"newzealand", "ngerman", "ngermanb", "norsk", "nynorsk", "polutonikogreek", "polish",
-"portuges", "portuguese", "romanian", "russian", "russianb", "samin",
-"scottish", "serbian", "serbian-latin", "slovak", "slovene", "spanish",
-"swedish", "thai", "turkish", "turkmen", "ukraineb", "ukrainian",
-"uppersorbian", "UKenglish", "USenglish", "usorbian", "vietnam", "welsh",
+"french", "frenchb", "frenchle", "frenchpro", "friulan", "galician", "german", "germanb",
+"georgian", "greek", "hebrew", "hungarian", "icelandic", "indon", "indonesian",
+"interlingua", "irish", "italian", "japanese", "kazakh", "kurmanji", "latin",
+"latvian", "lithuanian", "lowersorbian", "lsorbian", "macedonian", "magyar", "malay", "meyalu",
+"mongolian", "naustrian", "newzealand", "ngerman", "ngermanb", "norsk", "nswissgerman",
+"nynorsk", "piedmontese", "polutonikogreek", "polish", "portuges", "portuguese",
+"romanian", "romansh", "russian", "russianb", "samin", "scottish", "serbian", "serbian-latin",
+"slovak", "slovene", "spanish", "swedish", "swissgerman", "thai", "turkish", "turkmen",
+"ukraineb", "ukrainian", "uppersorbian", "UKenglish", "USenglish", "usorbian",
+"vietnam", "welsh",
 0};
 
 /**
@@ -74,20 +73,24 @@ const char * const known_languages[] = {"acadian", "afrikaans", "albanian",
  */
 const char * const known_coded_languages[] = {"french", "afrikaans", "albanian",
 "american", "arabic_arabi", "arabic_arabtex", "australian", "austrian", "bahasa", "bahasa",
-"bahasam", "basque", "belarusian", "brazilian", "brazilian", "breton", "british",
+"bahasam", "basque", "belarusian", "bosnian", "brazilian", "brazilian", "breton", "british",
 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "french",
-"french", "french", "french", "french", "galician", "german", "german",
-"greek", "hebrew", "magyar", "icelandic", "bahasa", "bahasa", "interlingua",
-"irish", "italian", "japanese", "kazakh", "kurmanji", "latin", "latvian", "lithuanian",
-"lowersorbian", "lowersorbian", "magyar", "bahasam", "bahasam", "mongolian", "naustrian",
-"newzealand", "ngerman", "ngerman", "norsk", "nynorsk", "polutonikogreek", "polish",
-"portuguese", "portuguese", "romanian", "russian", "russian", "samin",
-"scottish", "serbian", "serbian-latin", "slovak", "slovene", "spanish",
-"swedish", "thai", "turkish", "turkmen", "ukrainian", "ukrainian",
-"uppersorbian", "uppersorbian", "english", "english", "vietnamese", "welsh",
+"french", "french", "french", "french", "friulan", "galician", "german", "german",
+"georgian", "greek", "hebrew", "magyar", "icelandic", "bahasa", "bahasa",
+"interlingua", "irish", "italian", "japanese", "kazakh", "kurmanji", "latin",
+"latvian", "lithuanian", "lowersorbian", "lowersorbian", "macedonian", "magyar", "bahasam", "bahasam",
+"mongolian", "naustrian", "newzealand", "ngerman", "ngerman", "norsk", "german-ch",
+"nynorsk", "piedmontese", "polutonikogreek", "polish", "portuguese", "portuguese",
+"romanian", "romansh", "russian", "russian", "samin", "scottish", "serbian", "serbian-latin",
+"slovak", "slovene", "spanish", "swedish", "german-ch-old", "thai", "turkish", "turkmen",
+"ukrainian", "ukrainian", "uppersorbian", "english", "english", "uppersorbian",
+"vietnamese", "welsh",
 0};
 
+/// languages with danish quotes (.lyx names)
+const char * const known_danish_quotes_languages[] = {"danish", 0};
+
 /// languages with english quotes (.lyx names)
 const char * const known_english_quotes_languages[] = {"american", "australian",
 "bahasa", "bahasam", "brazilian", "canadian", "chinese-simplified", "english",
@@ -96,18 +99,19 @@ const char * const known_english_quotes_languages[] = {"american", "australian",
 
 /// languages with french quotes (.lyx names)
 const char * const known_french_quotes_languages[] = {"albanian",
-"arabic_arabi", "arabic_arabtex", "basque", "canadien", "catalan", "french",
-"galician", "greek", "italian", "norsk", "nynorsk", "polutonikogreek",
-"russian", "spanish", "spanish-mexico", "turkish", "turkmen", "ukrainian",
-"vietnamese", 0};
+"arabic_arabi", "arabic_arabtex", "asturian", "basque", "canadien", "catalan",
+"french", "friulan", "galician", "greek", "italian", "norsk", "nynorsk",
+"piedmontese", "polutonikogreek", "russian", "spanish", "spanish-mexico",
+"turkish", "turkmen", "ukrainian", "vietnamese", 0};
 
 /// languages with german quotes (.lyx names)
 const char * const known_german_quotes_languages[] = {"austrian", "bulgarian",
-"czech", "german", "icelandic", "lithuanian", "lowersorbian", "naustrian",
-"ngerman", "serbian", "serbian-latin", "slovak", "slovene", "uppersorbian", 0};
+"czech", "german", "georgian", "icelandic", "lithuanian", "lowersorbian", "macedonian",
+"naustrian", "ngerman", "romansh", "serbian", "serbian-latin", "slovak", "slovene",
+"uppersorbian", 0};
 
 /// languages with polish quotes (.lyx names)
-const char * const known_polish_quotes_languages[] = {"afrikaans", "croatian",
+const char * const known_polish_quotes_languages[] = {"afrikaans", "bosnian", "croatian",
 "dutch", "estonian", "magyar", "polish", "romanian", 0};
 
 /// languages with swedish quotes (.lyx names)
@@ -121,15 +125,20 @@ const char * const known_old_language_packages[] = {"french", "frenchle",
 char const * const known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
 
 const char * const known_roman_fonts[] = { "ae", "beraserif", "bookman",
-"ccfonts", "chancery", "charter", "cmr", "fourier", "lmodern", "mathpazo",
-"mathptmx", "newcent", "utopia", 0};
+"ccfonts", "chancery", "charter", "cmr", "cochineal", "crimson", "fourier",
+"garamondx", "libertine", "libertine-type1", "lmodern", "mathdesign", "mathpazo",
+"mathptmx", "newcent", "NotoSerif-TLF", "tgbonum", "tgchorus", "tgpagella", "tgschola",
+"tgtermes", "utopia", 0};
 
-const char * const known_sans_fonts[] = { "avant", "berasans", "cmbr", "cmss",
-"helvet", "lmss", 0};
+const char * const known_sans_fonts[] = { "avant", "berasans", "biolinum-type1",
+"cmbr", "cmss", "helvet", "iwona", "iwonac", "iwonal", "iwonalc", "kurier",
+"kurierc", "kurierl", "kurierlc", "lmss", "NotoSans-TLF", "tgadventor", "tgheros", 0};
 
 const char * const known_typewriter_fonts[] = { "beramono", "cmtl", "cmtt",
-"courier", "lmtt", "luximono", "fourier", "lmodern", "mathpazo", "mathptmx",
-"newcent", 0};
+"courier", "lmtt", "luximono", "fourier", "libertineMono-type1", "lmodern",
+"mathpazo", "mathptmx", "newcent", "NotoMono-TLF", "tgcursor", "txtt", 0};
+
+const char * const known_math_fonts[] = { "eulervm", "newtxmath", 0};
 
 const char * const known_paper_sizes[] = { "a0paper", "b0paper", "c0paper",
 "a1paper", "b1paper", "c1paper", "a2paper", "b2paper", "c2paper", "a3paper",
@@ -152,11 +161,13 @@ const char * const known_if_commands[] = {"if", "ifarydshln", "ifbraket",
 "ifcancel", "ifcolortbl", "ifeurosym", "ifmarginnote", "ifmmode", "ifpdf",
 "ifsidecap", "ifupgreek", 0};
 
-const char * const known_basic_colors[] = {"blue", "black", "cyan", "green",
-"magenta", "red", "white", "yellow", 0};
+const char * const known_basic_colors[] = {"black", "blue", "brown", "cyan",
+       "darkgray", "gray", "green", "lightgray", "lime", "magenta", "orange", "olive",
+       "pink", "purple", "red", "teal", "violet", "white", "yellow", 0};
 
-const char * const known_basic_color_codes[] = {"#0000ff", "#000000", "#00ffff", "#00ff00",
-"#ff00ff", "#ff0000", "#ffffff", "#ffff00", 0};
+const char * const known_basic_color_codes[] = {"#000000", "#0000ff", "#964B00", "#00ffff",
+       "#a9a9a9", "#808080", "#00ff00", "#d3d3d3", "#bfff00", "#ff00ff", "#ff7f00", "#808000",
+       "#ffc0cb", "#800080", "#ff0000", "#008080", "#8f00ff", "#ffffff", "#ffff00", 0};
 
 /// conditional commands with three arguments like \@ifundefined{}{}{}
 const char * const known_if_3arg_commands[] = {"@ifundefined", "IfFileExists",
@@ -170,11 +181,11 @@ const char * const known_xetex_packages[] = {"arabxetex", "fixlatvian",
 
 /// packages that are automatically skipped if loaded by LyX
 const char * const known_lyx_packages[] = {"amsbsy", "amsmath", "amssymb",
-"amstext", "amsthm", "array", "babel", "booktabs", "calc", "CJK", "color", "float",
-"fontspec", "graphicx", "hhline", "ifthen", "longtable", "makeidx", "multirow",
-"nomencl", "pdfpages", "rotating", "rotfloat", "splitidx", "setspace",
-"subscript", "textcomp", "ulem", "url", "varioref", "verbatim", "wrapfig",
-"xunicode", 0};
+"amstext", "amsthm", "array", "babel", "booktabs", "calc", "CJK", "color",
+"float", "fontspec", "framed", "graphicx", "hhline", "ifthen", "longtable",
+"makeidx", "multirow", "nomencl", "pdfpages", "prettyref", "refstyle", "rotating",
+"rotfloat", "splitidx", "setspace", "subscript", "textcomp", "tipa", "tipx",
+"tone", "ulem", "url", "varioref", "verbatim", "wrapfig", "xcolor", "xunicode", 0};
 
 // codes used to remove packages that are loaded automatically by LyX.
 // Syntax: package_beg_sep<name>package_mid_sep<package loading code>package_end_sep
@@ -284,38 +295,47 @@ string process_keyval_opt(vector<string> & options, string name)
 
 /**
  * known polyglossia language names (including variants)
+ * FIXME: support spelling=old for german variants (german vs. ngerman LyX names etc)
  */
 const char * const Preamble::polyglossia_languages[] = {
-"albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
-"nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
-"armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
-"irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
-"bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
-"samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
-"brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
-"lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
-"welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
-"marathi", "spanish",
-"american", "ancient", "australian", "british", "monotonic", "newzealand",
-"polytonic", 0};
+"albanian", "american", "amharic", "ancient", "arabic", "armenian", "asturian", "australian",
+"bahasai", "bahasam", "basque", "bengali", "brazil", "brazilian", "breton", "british", "bulgarian",
+"catalan", "coptic", "croatian", "czech", "danish", "divehi", "dutch",
+"english", "esperanto", "estonian", "farsi", "finnish", "french", "friulan",
+"galician", "greek", "monotonic", "hebrew", "hindi",
+"icelandic", "interlingua", "irish", "italian", "kannada", "khmer",
+"lao", "latin", "latvian", "lithuanian", "lsorbian", "magyar", "malayalam", "marathi",
+"austrian", "newzealand", "german", "norsk", "nynorsk", "occitan",
+"piedmontese", "polish", "polytonic", "portuges", "romanian", "romansh", "russian",
+"samin", "sanskrit", "scottish", "serbian", "slovak", "slovenian", "spanish", "swedish", "syriac",
+"tamil", "telugu", "thai", "tibetan", "turkish", "turkmen",
+"ukrainian", "urdu", "usorbian", "vietnamese", "welsh", 0};
+// not yet supported by LyX: "korean", "nko"
 
 /**
  * the same as polyglossia_languages with .lyx names
  * please keep this in sync with polyglossia_languages line by line!
  */
 const char * const Preamble::coded_polyglossia_languages[] = {
-"albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
-"nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
-"armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
-"irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
-"bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
-"samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
-"brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
-"lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
-"welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
-"marathi", "spanish",
-"american", "ancientgreek", "australian", "british", "greek", "newzealand",
-"polutonikogreek", 0};
+"albanian", "american", "amharic", "ancientgreek", "arabic_arabi", "armenian", "asturian", "australian",
+"bahasa", "bahasam", "basque", "bengali", "brazilian", "brazilian", "breton", "british", "bulgarian",
+"catalan", "coptic", "croatian", "czech", "danish", "divehi", "dutch",
+"english", "esperanto", "estonian", "farsi", "finnish", "french", "friulan",
+"galician", "greek", "greek", "hebrew", "hindi",
+"icelandic", "interlingua", "irish", "italian", "kannada", "khmer",
+"lao", "latin", "latvian", "lithuanian", "lowersorbian", "magyar", "malayalam", "marathi",
+"naustrian","newzealand", "ngerman", "norsk", "nynorsk", "occitan",
+"piedmontese", "polish", "polutonikogreek", "portuges", "romanian", "romansh", "russian",
+"samin", "sanskrit", "scottish", "serbian", "slovak", "slovene", "spanish", "swedish", "syriac",
+"tamil", "telugu", "thai", "tibetan", "turkish", "turkmen",
+"ukrainian", "urdu", "uppersorbian", "vietnamese", "welsh", 0};
+// not yet supported by LyX: "korean-polyglossia", "nko"
+
+
+bool Preamble::usePolyglossia() const
+{
+       return h_use_non_tex_fonts && h_language_package == "default";
+}
 
 
 bool Preamble::indentParagraphs() const
@@ -382,6 +402,15 @@ Author const & Preamble::getAuthor(std::string const & name) const
 }
 
 
+int Preamble::getSpecialTableColumnArguments(char c) const
+{
+       map<char, int>::const_iterator it = special_columns_.find(c);
+       if (it == special_columns_.end())
+               return -1;
+       return it->second;
+}
+
+
 void Preamble::add_package(string const & name, vector<string> & options)
 {
        // every package inherits the global options
@@ -403,18 +432,19 @@ void Preamble::add_package(string const & name, vector<string> & options)
 
 namespace {
 
-// Given is a string like "scaled=0.9", return 0.9 * 100
-string const scale_as_percentage(string const & scale)
+// Given is a string like "scaled=0.9" or "Scale=0.9", return 0.9 * 100
+bool scale_as_percentage(string const & scale, string & percentage)
 {
        string::size_type pos = scale.find('=');
        if (pos != string::npos) {
                string value = scale.substr(pos + 1);
-               if (isStrDbl(value))
-                       return convert<string>(100 * convert<double>(value));
+               if (isStrDbl(value)) {
+                       percentage = convert<string>(
+                               static_cast<int>(100 * convert<double>(value)));
+                       return true;
+               }
        }
-       // If the input string didn't match our expectations.
-       // return the default value "100"
-       return "100";
+       return false;
 }
 
 
@@ -430,35 +460,46 @@ string remove_braces(string const & value)
 } // anonymous namespace
 
 
-Preamble::Preamble() : one_language(true), title_layout_found(false)
+Preamble::Preamble() : one_language(true), explicit_babel(false),
+       title_layout_found(false), index_number(0), h_font_cjk_set(false),
+       h_use_microtype("false")
 {
        //h_backgroundcolor;
        //h_boxbgcolor;
        h_biblio_style            = "plain";
        h_bibtex_command          = "default";
        h_cite_engine             = "basic";
-       h_cite_engine_type        = "numerical";
+       h_cite_engine_type        = "default";
        h_color                   = "#008000";
        h_defskip                 = "medskip";
+       h_dynamic_quotes          = false;
        //h_float_placement;
        //h_fontcolor;
        h_fontencoding            = "default";
-       h_font_roman              = "default";
-       h_font_sans               = "default";
-       h_font_typewriter         = "default";
-       h_font_math               = "auto";
+       h_font_roman[0]           = "default";
+       h_font_roman[1]           = "default";
+       h_font_sans[0]            = "default";
+       h_font_sans[1]            = "default";
+       h_font_typewriter[0]      = "default";
+       h_font_typewriter[1]      = "default";
+       h_font_math[0]            = "auto";
+       h_font_math[1]            = "auto";
        h_font_default_family     = "default";
-       h_use_non_tex_fonts       = "false";
+       h_use_non_tex_fonts       = false;
        h_font_sc                 = "false";
        h_font_osf                = "false";
-       h_font_sf_scale           = "100";
-       h_font_tt_scale           = "100";
+       h_font_sf_scale[0]        = "100";
+       h_font_sf_scale[1]        = "100";
+       h_font_tt_scale[0]        = "100";
+       h_font_tt_scale[1]        = "100";
+       //h_font_cjk
+       h_is_mathindent           = "0";
        h_graphics                = "default";
        h_default_output_format   = "default";
        h_html_be_strict          = "false";
        h_html_css_as_file        = "0";
        h_html_math_output        = "0";
-       h_index                   = "Index";
+       h_index[0]                = "Index";
        h_index_command           = "default";
        h_inputencoding           = "auto";
        h_justification           = "true";
@@ -484,7 +525,7 @@ Preamble::Preamble() : one_language(true), title_layout_found(false)
        //h_pdf_author;
        //h_pdf_subject;
        //h_pdf_keywords;
-       h_pdf_bookmarks           = "1";
+       h_pdf_bookmarks           = "0";
        h_pdf_bookmarksnumbered   = "0";
        h_pdf_bookmarksopen       = "0";
        h_pdf_bookmarksopenlevel  = "1";
@@ -492,29 +533,35 @@ Preamble::Preamble() : one_language(true), title_layout_found(false)
        h_pdf_pdfborder           = "0";
        h_pdf_colorlinks          = "0";
        h_pdf_backref             = "section";
-       h_pdf_pdfusetitle         = "1";
+       h_pdf_pdfusetitle         = "0";
        //h_pdf_pagemode;
        //h_pdf_quoted_options;
-       h_quotes_language         = "english";
+       h_quotes_style         = "english";
        h_secnumdepth             = "3";
-       h_shortcut                = "idx";
+       h_shortcut[0]             = "idx";
        h_spacing                 = "single";
+       h_save_transient_properties = "true";
        h_suppress_date           = "false";
        h_textclass               = "article";
        h_tocdepth                = "3";
        h_tracking_changes        = "false";
        h_use_bibtopic            = "false";
+       h_use_dash_ligatures      = "true";
        h_use_indices             = "false";
        h_use_geometry            = "false";
        h_use_default_options     = "false";
        h_use_hyperref            = "false";
-       h_use_refstyle            = "0";
+       h_use_microtype           = "false";
+       h_use_refstyle            = false;
        h_use_packages["amsmath"]    = "1";
        h_use_packages["amssymb"]    = "0";
+       h_use_packages["cancel"]     = "0";
        h_use_packages["esint"]      = "1";
        h_use_packages["mhchem"]     = "0";
        h_use_packages["mathdots"]   = "0";
        h_use_packages["mathtools"]  = "0";
+       h_use_packages["stackrel"]   = "0";
+       h_use_packages["stmaryrd"]   = "0";
        h_use_packages["undertilde"] = "0";
 }
 
@@ -634,72 +681,180 @@ void Preamble::handle_geometry(vector<string> & options)
 
 
 void Preamble::handle_package(Parser &p, string const & name,
-                              string const & opts, bool in_lyx_preamble)
+                              string const & opts, bool in_lyx_preamble,
+                              bool detectEncoding)
 {
        vector<string> options = split_options(opts);
        add_package(name, options);
-       string scale;
+       char const * const * where = 0;
 
        if (is_known(name, known_xetex_packages)) {
                xetex = true;
-               h_use_non_tex_fonts = "true";
+               h_use_non_tex_fonts = true;
                registerAutomaticallyLoadedPackage("fontspec");
                if (h_inputencoding == "auto")
-                       p.setEncoding("utf8");
+                       p.setEncoding("UTF-8");
        }
 
        // roman fonts
        if (is_known(name, known_roman_fonts))
-               h_font_roman = name;
+               h_font_roman[0] = name;
 
        if (name == "fourier") {
-               h_font_roman = "utopia";
+               h_font_roman[0] = "utopia";
                // when font uses real small capitals
                if (opts == "expert")
                        h_font_sc = "true";
        }
 
+       if (name == "garamondx") {
+               h_font_roman[0] = "garamondx";
+               if (opts == "osfI")
+                       h_font_osf = "true";
+       }
+
+       if (name == "libertine") {
+               h_font_roman[0] = "libertine";
+               // this automatically invokes biolinum
+               h_font_sans[0] = "biolinum";
+               if (opts == "osf")
+                       h_font_osf = "true";
+               else if (opts == "lining")
+                       h_font_osf = "false";
+       }
+
+       if (name == "libertine-type1") {
+               h_font_roman[0] = "libertine";
+               // NOTE: contrary to libertine.sty, libertine-type1
+               // does not automatically invoke biolinum
+               if (opts == "lining")
+                       h_font_osf = "false";
+               else if (opts == "osf")
+                       h_font_osf = "true";
+       }
+
+       if (name == "mathdesign") {
+               if (opts.find("charter") != string::npos)
+                       h_font_roman[0] = "md-charter";
+               if (opts.find("garamond") != string::npos)
+                       h_font_roman[0] = "md-garamond";
+               if (opts.find("utopia") != string::npos)
+                       h_font_roman[0] = "md-utopia";
+               if (opts.find("expert") != string::npos) {
+                       h_font_sc = "true";
+                       h_font_osf = "true";
+               }
+       }
+
        else if (name == "mathpazo")
-               h_font_roman = "palatino";
+               h_font_roman[0] = "palatino";
 
        else if (name == "mathptmx")
-               h_font_roman = "times";
+               h_font_roman[0] = "times";
+
+       if (name == "crimson")
+               h_font_roman[0] = "cochineal";
+
+       if (name == "cochineal") {
+               h_font_roman[0] = "cochineal";
+               // cochineal can have several options, e.g. [proportional,osf]
+               string::size_type pos = opts.find("osf");
+               if (pos != string::npos)
+                       h_font_osf = "true";
+       }
+
+       if (name == "noto") {
+               // noto can have several options
+               if (opts.empty())
+                       h_font_roman[0] = "NotoSerif-TLF";
+               string::size_type pos = opts.find("rm");
+               if (pos != string::npos)
+                       h_font_roman[0] = "NotoSerif-TLF";
+               pos = opts.find("sf");
+               if (pos != string::npos)
+                       h_font_sans[0] = "NotoSans-TLF";
+               pos = opts.find("nott");
+               if (pos != string::npos) {
+                       h_font_roman[0] = "NotoSerif-TLF";
+                       h_font_sans[0] = "NotoSans-TLF";
+               }
+               // noto as typewriter is handled in handling of \ttdefault
+               // special cases are handled in handling of \rmdefault and \sfdefault
+       }
 
        // sansserif fonts
        if (is_known(name, known_sans_fonts)) {
-               h_font_sans = name;
-               if (!opts.empty()) {
-                       scale = opts;
-                       h_font_sf_scale = scale_as_percentage(scale);
+               h_font_sans[0] = name;
+               if (options.size() >= 1) {
+                       if (scale_as_percentage(opts, h_font_sf_scale[0]))
+                               options.clear();
                }
        }
 
+       if (name == "biolinum-type1") {
+               h_font_sans[0] = "biolinum";
+               // biolinum can have several options, e.g. [osf,scaled=0.97]
+               string::size_type pos = opts.find("osf");
+               if (pos != string::npos)
+                       h_font_osf = "true";
+       }
+
        // typewriter fonts
        if (is_known(name, known_typewriter_fonts)) {
                // fourier can be set as roman font _only_
                // fourier as typewriter is handled in handling of \ttdefault
                if (name != "fourier") {
-                       h_font_typewriter = name;
-                       if (!opts.empty()) {
-                               scale = opts;
-                               h_font_tt_scale = scale_as_percentage(scale);
+                       h_font_typewriter[0] = name;
+                       if (options.size() >= 1) {
+                               if (scale_as_percentage(opts, h_font_tt_scale[0]))
+                                       options.clear();
                        }
                }
        }
 
+       if (name == "libertineMono-type1") {
+               h_font_typewriter[0] = "libertine-mono";
+       }
+
        // font uses old-style figure
        if (name == "eco")
                h_font_osf = "true";
 
+       // math fonts
+       if (is_known(name, known_math_fonts))
+               h_font_math[0] = name;
+
+       if (name == "newtxmath") {
+               if (opts.empty())
+                       h_font_math[0] = "newtxmath";
+               else if (opts == "garamondx")
+                       h_font_math[0] = "garamondx-ntxm";
+               else if (opts == "libertine")
+                       h_font_math[0] = "libertine-ntxm";
+               else if (opts == "minion")
+                       h_font_math[0] = "minion-ntxm";
+               else if (opts == "cochineal")
+                       h_font_math[0] = "cochineal-ntxm";
+       }
+
+       if (name == "iwona")
+               if (opts == "math")
+                       h_font_math[0] = "iwona-math";
+
+       if (name == "kurier")
+               if (opts == "math")
+                       h_font_math[0] = "kurier-math";
+
        // after the detection and handling of special cases, we can remove the
        // fonts, otherwise they would appear in the preamble, see bug #7856
        if (is_known(name, known_roman_fonts) || is_known(name, known_sans_fonts)
-               ||      is_known(name, known_typewriter_fonts))
+               ||      is_known(name, known_typewriter_fonts) || is_known(name, known_math_fonts))
                ;
-
-       else if (name == "amsmath" || name == "amssymb" ||
+       //"On". See the enum Package in BufferParams.h if you thought that "2" should have been "42"
+       else if (name == "amsmath" || name == "amssymb" || name == "cancel" ||
                 name == "esint" || name == "mhchem" || name == "mathdots" ||
-                name == "mathtools" || name == "undertilde")
+                name == "mathtools" || name == "stackrel" ||
+                name == "stmaryrd" || name == "undertilde")
                h_use_packages[name] = "2";
 
        else if (name == "babel") {
@@ -719,11 +874,19 @@ void Preamble::handle_package(Parser &p, string const & name,
                        handle_opt(options, known_languages, h_language);
                        // translate the babel name to a LyX name
                        h_language = babel2lyx(h_language);
-                       // for Japanese we assume EUC-JP as encoding
-                       // but we cannot determine the exact encoding and thus output also a note
                        if (h_language == "japanese") {
-                               h_inputencoding = "euc";
-                               p.setEncoding("EUC-JP");
+                               // For Japanese, the encoding isn't indicated in the source
+                               // file, and there's really not much we can do. We could
+                               // 1) offer a list of possible encodings to choose from, or
+                               // 2) determine the encoding of the file by inspecting it.
+                               // For the time being, we leave the encoding alone so that
+                               // we don't get iconv errors when making a wrong guess, and
+                               // we will output a note at the top of the document
+                               // explaining what to do.
+                               Encoding const * const enc = encodings.fromIconvName(
+                                       p.getEncoding(), Encoding::japanese, false);
+                               if (enc)
+                                       h_inputencoding = enc->name();
                                is_nonCJKJapanese = true;
                                // in this case babel can be removed from the preamble
                                registerAutomaticallyLoadedPackage("babel");
@@ -737,25 +900,23 @@ void Preamble::handle_package(Parser &p, string const & name,
                                h_preamble << "\\usepackage[" << opts << "]{babel}\n";
                        }
                        delete_opt(options, known_languages);
-               }
-               else
+               } else {
                        h_preamble << "\\usepackage{babel}\n";
+                       explicit_babel = true;
+               }
        }
 
        else if (name == "polyglossia") {
                h_language_package = "default";
                h_default_output_format = "pdf4";
-               h_use_non_tex_fonts = "true";
+               h_use_non_tex_fonts = true;
                xetex = true;
                registerAutomaticallyLoadedPackage("xunicode");
                if (h_inputencoding == "auto")
-                       p.setEncoding("utf8");
+                       p.setEncoding("UTF-8");
        }
 
        else if (name == "CJK") {
-               // It is impossible to determine the document language if CJK is used.
-               // All we can do is to notify the user that he has to set this by himself.
-               have_CJK = true;
                // set the encoding to "auto" because it might be set to "default" by the babel handling
                // and this would not be correct for CJK
                if (h_inputencoding == "default")
@@ -763,6 +924,12 @@ void Preamble::handle_package(Parser &p, string const & name,
                registerAutomaticallyLoadedPackage("CJK");
        }
 
+       else if (name == "CJKutf8") {
+               h_inputencoding = "utf8-cjk";
+               p.setEncoding("UTF-8");
+               registerAutomaticallyLoadedPackage("CJKutf8");
+       }
+
        else if (name == "fontenc") {
                h_fontencoding = getStringFromVector(options, ",");
                /* We could do the following for better round trip support,
@@ -777,14 +944,24 @@ void Preamble::handle_package(Parser &p, string const & name,
                // h_inputencoding is only set when there is not more than one
                // inputenc option because otherwise h_inputencoding must be
                // set to "auto" (the default encoding of the document language)
-               // Therefore check for the "," character.
+               // Therefore check that exactly one option is passed to inputenc.
                // It is also only set when there is not more than one babel
                // language option.
-               if (opts.find(",") == string::npos && one_language == true)
-                       h_inputencoding = opts;
-               if (!options.empty())
-                       p.setEncoding(options.back());
-               options.clear();
+               if (!options.empty()) {
+                       string const encoding = options.back();
+                       Encoding const * const enc = encodings.fromLaTeXName(
+                               encoding, Encoding::inputenc, true);
+                       if (!enc) {
+                               if (!detectEncoding)
+                                       cerr << "Unknown encoding " << encoding
+                                            << ". Ignoring." << std::endl;
+                       } else {
+                               if (!enc->unsafe() && options.size() == 1 && one_language == true)
+                                       h_inputencoding = enc->name();
+                               p.setEncoding(enc->iconvName());
+                       }
+                       options.clear();
+               }
        }
 
        else if (name == "srcltx") {
@@ -803,9 +980,6 @@ void Preamble::handle_package(Parser &p, string const & name,
                h_language_package = "\\usepackage{" + name + "}";
        }
 
-       else if (name == "prettyref")
-               ; // ignore this FIXME: Use the package separator mechanism instead
-
        else if (name == "lyxskak") {
                // ignore this and its options
                const char * const o[] = {"ps", "mover", 0};
@@ -815,6 +989,10 @@ void Preamble::handle_package(Parser &p, string const & name,
        else if (is_known(name, known_lyx_packages) && options.empty()) {
                if (name == "splitidx")
                        h_use_indices = "true";
+               if (name == "refstyle")
+                       h_use_refstyle = true;
+               else if (name == "prettyref")
+                       h_use_refstyle = false;
                if (!in_lyx_preamble) {
                        h_preamble << package_beg_sep << name
                                   << package_mid_sep << "\\usepackage{"
@@ -833,8 +1011,8 @@ void Preamble::handle_package(Parser &p, string const & name,
        else if (name == "subfig")
                ; // ignore this FIXME: Use the package separator mechanism instead
 
-       else if (is_known(name, known_languages))
-               h_language = name;
+       else if ((where = is_known(name, known_languages)))
+               h_language = known_coded_languages[where - known_languages];
 
        else if (name == "natbib") {
                h_biblio_style = "plainnat";
@@ -859,9 +1037,31 @@ void Preamble::handle_package(Parser &p, string const & name,
                h_cite_engine_type = "authoryear";
        }
 
+       else if (name == "bibtopic")
+               h_use_bibtopic = "true";
+
        else if (name == "hyperref")
                handle_hyperref(options);
 
+       else if (name == "algorithm2e") {
+               // Load "algorithm2e" module
+               addModule("algorithm2e");
+               // Add the package options to the global document options
+               if (!options.empty()) {
+                       if (h_options.empty())
+                               h_options = join(options, ",");
+                       else
+                               h_options += ',' + join(options, ",");
+               }
+       }
+       else if (name == "microtype") {
+               //we internally support only microtype without params
+               if (options.empty())
+                       h_use_microtype = "true";
+               else
+                       h_preamble << "\\usepackage[" << opts << "]{microtype}";
+       }
+
        else if (!in_lyx_preamble) {
                if (options.empty())
                        h_preamble << "\\usepackage{" << name << '}';
@@ -877,7 +1077,7 @@ void Preamble::handle_package(Parser &p, string const & name,
        }
 
        // We need to do something with the options...
-       if (!options.empty())
+       if (!options.empty() && !detectEncoding)
                cerr << "Ignoring options '" << join(options, ",")
                     << "' of package " << name << '.' << endl;
 
@@ -903,7 +1103,7 @@ void Preamble::handle_if(Parser & p, bool in_lyx_preamble)
 }
 
 
-bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
+bool Preamble::writeLyXHeader(ostream & os, bool subdoc, string const & outfiledir)
 {
        // set the quote language
        // LyX only knows the following quotes languages:
@@ -914,23 +1114,23 @@ bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
        // http://en.wikipedia.org/wiki/Quotation_mark,_non-English_usage
        // (quotes for kazakh and interlingua are unknown)
        // danish
-       if (h_language == "danish")
-               h_quotes_language = "danish";
+       if (is_known(h_language, known_danish_quotes_languages))
+               h_quotes_style = "danish";
        // french
        else if (is_known(h_language, known_french_quotes_languages))
-               h_quotes_language = "french";
+               h_quotes_style = "french";
        // german
        else if (is_known(h_language, known_german_quotes_languages))
-               h_quotes_language = "german";
+               h_quotes_style = "german";
        // polish
        else if (is_known(h_language, known_polish_quotes_languages))
-               h_quotes_language = "polish";
+               h_quotes_style = "polish";
        // swedish
        else if (is_known(h_language, known_swedish_quotes_languages))
-               h_quotes_language = "swedish";
+               h_quotes_style = "swedish";
        //english
        else if (is_known(h_language, known_english_quotes_languages))
-               h_quotes_language = "english";
+               h_quotes_style = "english";
 
        if (contains(h_float_placement, "H"))
                registerAutomaticallyLoadedPackage("float");
@@ -943,10 +1143,16 @@ bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
        }
 
        // output the LyX file settings
-       os << "#LyX file created by tex2lyx " << PACKAGE_VERSION << "\n"
+       // Important: Keep the version formatting in sync with LyX and
+       //            lyx2lyx (bug 7951)
+       string const origin = roundtripMode() ? "roundtrip" : outfiledir;
+       os << "#LyX file created by tex2lyx " << lyx_version_major << '.'
+          << lyx_version_minor << '\n'
           << "\\lyxformat " << LYX_FORMAT << '\n'
           << "\\begin_document\n"
           << "\\begin_header\n"
+          << "\\save_transient_properties " << h_save_transient_properties << "\n"
+          << "\\origin " << origin << "\n"
           << "\\textclass " << h_textclass << "\n";
        string const raw = subdoc ? empty_string() : h_preamble.str();
        if (!raw.empty()) {
@@ -988,17 +1194,25 @@ bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
           << "\\language_package " << h_language_package << "\n"
           << "\\inputencoding " << h_inputencoding << "\n"
           << "\\fontencoding " << h_fontencoding << "\n"
-          << "\\font_roman " << h_font_roman << "\n"
-          << "\\font_sans " << h_font_sans << "\n"
-          << "\\font_typewriter " << h_font_typewriter << "\n"
-          << "\\font_math " << h_font_math << "\n"
+          << "\\font_roman \"" << h_font_roman[0]
+          << "\" \"" << h_font_roman[1] << "\"\n"
+          << "\\font_sans \"" << h_font_sans[0] << "\" \"" << h_font_sans[1] << "\"\n"
+          << "\\font_typewriter \"" << h_font_typewriter[0]
+          << "\" \"" << h_font_typewriter[1] << "\"\n"
+          << "\\font_math \"" << h_font_math[0] << "\" \"" << h_font_math[1] << "\"\n"
           << "\\font_default_family " << h_font_default_family << "\n"
-          << "\\use_non_tex_fonts " << h_use_non_tex_fonts << "\n"
+          << "\\use_non_tex_fonts " << (h_use_non_tex_fonts ? "true" : "false") << '\n'
           << "\\font_sc " << h_font_sc << "\n"
           << "\\font_osf " << h_font_osf << "\n"
-          << "\\font_sf_scale " << h_font_sf_scale << "\n"
-          << "\\font_tt_scale " << h_font_tt_scale << "\n"
-          << "\\graphics " << h_graphics << "\n"
+          << "\\font_sf_scale " << h_font_sf_scale[0]
+          << ' ' << h_font_sf_scale[1] << '\n'
+          << "\\font_tt_scale " << h_font_tt_scale[0]
+          << ' ' << h_font_tt_scale[1] << '\n';
+       if (!h_font_cjk.empty())
+               os << "\\font_cjk " << h_font_cjk << '\n';
+       os << "\\use_microtype " << h_use_microtype << '\n'
+          << "\\use_dash_ligatures " << h_use_dash_ligatures << '\n'
+          << "\\graphics " << h_graphics << '\n'
           << "\\default_output_format " << h_default_output_format << "\n"
           << "\\output_sync " << h_output_sync << "\n";
        if (h_output_sync == "1")
@@ -1012,13 +1226,13 @@ bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
           << "\\use_hyperref " << h_use_hyperref << '\n';
        if (h_use_hyperref == "true") {
                if (!h_pdf_title.empty())
-                       os << "\\pdf_title \"" << h_pdf_title << "\"\n";
+                       os << "\\pdf_title " << Lexer::quoteString(h_pdf_title) << '\n';
                if (!h_pdf_author.empty())
-                       os << "\\pdf_author \"" << h_pdf_author << "\"\n";
+                       os << "\\pdf_author " << Lexer::quoteString(h_pdf_author) << '\n';
                if (!h_pdf_subject.empty())
-                       os << "\\pdf_subject \"" << h_pdf_subject << "\"\n";
+                       os << "\\pdf_subject " << Lexer::quoteString(h_pdf_subject) << '\n';
                if (!h_pdf_keywords.empty())
-                       os << "\\pdf_keywords \"" << h_pdf_keywords << "\"\n";
+                       os << "\\pdf_keywords " << Lexer::quoteString(h_pdf_keywords) << '\n';
                os << "\\pdf_bookmarks " << h_pdf_bookmarks << "\n"
                      "\\pdf_bookmarksnumbered " << h_pdf_bookmarksnumbered << "\n"
                      "\\pdf_bookmarksopen " << h_pdf_bookmarksopen << "\n"
@@ -1031,7 +1245,7 @@ bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
                if (!h_pdf_pagemode.empty())
                        os << "\\pdf_pagemode " << h_pdf_pagemode << '\n';
                if (!h_pdf_quoted_options.empty())
-                       os << "\\pdf_quoted_options \"" << h_pdf_quoted_options << "\"\n";
+                       os << "\\pdf_quoted_options " << Lexer::quoteString(h_pdf_quoted_options) << '\n';
        }
        os << "\\papersize " << h_papersize << "\n"
           << "\\use_geometry " << h_use_geometry << '\n';
@@ -1055,10 +1269,19 @@ bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
                os << "\\backgroundcolor " << h_backgroundcolor << '\n';
        if (!h_boxbgcolor.empty())
                os << "\\boxbgcolor " << h_boxbgcolor << '\n';
-       os << "\\index " << h_index << '\n'
-          << "\\shortcut " << h_shortcut << '\n'
-          << "\\color " << h_color << '\n'
-          << "\\end_index\n";
+       if (index_number != 0)
+               for (int i = 0; i < index_number; i++) {
+                       os << "\\index " << h_index[i] << '\n'
+                          << "\\shortcut " << h_shortcut[i] << '\n'
+                          << "\\color " << h_color << '\n'
+                          << "\\end_index\n";
+               }
+       else {
+               os << "\\index " << h_index[0] << '\n'
+                  << "\\shortcut " << h_shortcut[0] << '\n'
+                  << "\\color " << h_color << '\n'
+                  << "\\end_index\n";
+       }
        os << h_margins
           << "\\secnumdepth " << h_secnumdepth << "\n"
           << "\\tocdepth " << h_tocdepth << "\n"
@@ -1067,7 +1290,11 @@ bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
                os << "\\defskip " << h_defskip << "\n";
        else
                os << "\\paragraph_indentation " << h_paragraph_indentation << "\n";
-       os << "\\quotes_language " << h_quotes_language << "\n"
+       os << "\\is_math_indent " << h_is_mathindent << "\n";
+       if (!h_mathindentation.empty())
+               os << "\\math_indentation " << h_mathindentation << "\n";
+       os << "\\quotes_style " << h_quotes_style << "\n"
+          << "\\dynamic_quotes " << h_dynamic_quotes << "\n"
           << "\\papercolumns " << h_papercolumns << "\n"
           << "\\papersides " << h_papersides << "\n"
           << "\\paperpagestyle " << h_paperpagestyle << "\n";
@@ -1089,7 +1316,14 @@ void Preamble::parse(Parser & p, string const & forceclass,
                      TeX2LyXDocClass & tc)
 {
        // initialize fixed types
-       special_columns['D'] = 3;
+       special_columns_['D'] = 3;
+       parse(p, forceclass, false, tc);
+}
+
+
+void Preamble::parse(Parser & p, string const & forceclass,
+                     bool detectEncoding, TeX2LyXDocClass & tc)
+{
        bool is_full_document = false;
        bool is_lyx_file = false;
        bool in_lyx_preamble = false;
@@ -1105,11 +1339,19 @@ void Preamble::parse(Parser & p, string const & forceclass,
        }
        p.reset();
 
+       if (detectEncoding && !is_full_document)
+               return;
+
        while (is_full_document && p.good()) {
+               if (detectEncoding && h_inputencoding != "auto" &&
+                   h_inputencoding != "default")
+                       return;
+
                Token const & t = p.get_token();
 
 #ifdef FILEDEBUG
-               cerr << "t: " << t << "\n";
+               if (!detectEncoding)
+                       cerr << "t: " << t << '\n';
 #endif
 
                //
@@ -1144,7 +1386,8 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        if (comment.size() > magicXeLaTeX.size()
                                  && comment.substr(0, magicXeLaTeX.size()) == magicXeLaTeX
                                  && h_inputencoding == "auto") {
-                               cerr << "XeLaTeX comment found, switching to UTF8\n";
+                               if (!detectEncoding)
+                                       cerr << "XeLaTeX comment found, switching to UTF8\n";
                                h_inputencoding = "utf8";
                        }
                        smatch sub;
@@ -1196,7 +1439,7 @@ void Preamble::parse(Parser & p, string const & forceclass,
                else if (t.cs() == "setmainfont") {
                        // we don't care about the option
                        p.hasOpt() ? p.getOpt() : string();
-                       h_font_roman = p.getArg('{', '}');
+                       h_font_roman[1] = p.getArg('{', '}');
                }
 
                else if (t.cs() == "setsansfont" || t.cs() == "setmonofont") {
@@ -1209,19 +1452,19 @@ void Preamble::parse(Parser & p, string const & forceclass,
                                if (pos != string::npos) {
                                        string::size_type i = fontopts.find(',', pos);
                                        if (i == string::npos)
-                                               scale = scale_as_percentage(fontopts.substr(pos + 1));
+                                               scale_as_percentage(fontopts.substr(pos + 1), scale);
                                        else
-                                               scale = scale_as_percentage(fontopts.substr(pos, i - pos));
+                                               scale_as_percentage(fontopts.substr(pos, i - pos), scale);
                                }
                        }
                        if (t.cs() == "setsansfont") {
                                if (!scale.empty())
-                                       h_font_sf_scale = scale;
-                               h_font_sans = p.getArg('{', '}');
+                                       h_font_sf_scale[1] = scale;
+                               h_font_sans[1] = p.getArg('{', '}');
                        } else {
                                if (!scale.empty())
-                                       h_font_tt_scale = scale;
-                               h_font_typewriter = p.getArg('{', '}');
+                                       h_font_tt_scale[1] = scale;
+                               h_font_typewriter[1] = p.getArg('{', '}');
                        }
                }
 
@@ -1240,9 +1483,9 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        // check the case that a standard color is used
                        if (space.empty() && is_known(argument, known_basic_colors)) {
                                h_fontcolor = rgbcolor2code(argument);
-                               preamble.registerAutomaticallyLoadedPackage("color");
+                               registerAutomaticallyLoadedPackage("color");
                        } else if (space.empty() && argument == "document_fontcolor")
-                               preamble.registerAutomaticallyLoadedPackage("color");
+                               registerAutomaticallyLoadedPackage("color");
                        // check the case that LyX's document_fontcolor is defined
                        // but not used for \color
                        else {
@@ -1262,7 +1505,7 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        if (is_known(argument, known_basic_colors)) {
                                h_backgroundcolor = rgbcolor2code(argument);
                        } else if (argument == "page_backgroundcolor")
-                               preamble.registerAutomaticallyLoadedPackage("color");
+                               registerAutomaticallyLoadedPackage("color");
                        // check the case that LyX's page_backgroundcolor is defined
                        // but not used for \pagecolor
                        else {
@@ -1275,12 +1518,75 @@ void Preamble::parse(Parser & p, string const & forceclass,
 
                else if (t.cs() == "makeatletter") {
                        // LyX takes care of this
-                       p.setCatCode('@', catLetter);
+                       p.setCatcode('@', catLetter);
                }
 
                else if (t.cs() == "makeatother") {
                        // LyX takes care of this
-                       p.setCatCode('@', catOther);
+                       p.setCatcode('@', catOther);
+               }
+
+               else if (t.cs() == "makeindex") {
+                       // LyX will re-add this if a print index command is found
+                       p.skip_spaces();
+               }
+
+               else if (t.cs() == "newindex") {
+                       string const indexname = p.getArg('[', ']');
+                       string const shortcut = p.verbatim_item();
+                       if (!indexname.empty())
+                               h_index[index_number] = indexname;
+                       else
+                               h_index[index_number] = shortcut;
+                       h_shortcut[index_number] = shortcut;
+                       index_number += 1;
+                       p.skip_spaces();
+               }
+
+               else if (t.cs() == "RS@ifundefined") {
+                       string const name = p.verbatim_item();
+                       string const body1 = p.verbatim_item();
+                       string const body2 = p.verbatim_item();
+                       // only non-lyxspecific stuff
+                       if (in_lyx_preamble &&
+                           (name == "subsecref" || name == "thmref" || name == "lemref"))
+                               p.skip_spaces();
+                       else {
+                               ostringstream ss;
+                               ss << '\\' << t.cs();
+                               ss << '{' << name << '}'
+                                  << '{' << body1 << '}'
+                                  << '{' << body2 << '}';
+                               h_preamble << ss.str();
+                       }
+               }
+
+               else if (t.cs() == "AtBeginDocument") {
+                       string const name = p.verbatim_item();
+                       // only non-lyxspecific stuff
+                       if (in_lyx_preamble &&
+                           (name == "\\providecommand\\partref[1]{\\ref{part:#1}}"
+                               || name == "\\providecommand\\chapref[1]{\\ref{chap:#1}}"
+                               || name == "\\providecommand\\secref[1]{\\ref{sec:#1}}"
+                               || name == "\\providecommand\\subsecref[1]{\\ref{subsec:#1}}"
+                               || name == "\\providecommand\\parref[1]{\\ref{par:#1}}"
+                               || name == "\\providecommand\\figref[1]{\\ref{fig:#1}}"
+                               || name == "\\providecommand\\tabref[1]{\\ref{tab:#1}}"
+                               || name == "\\providecommand\\algref[1]{\\ref{alg:#1}}"
+                               || name == "\\providecommand\\fnref[1]{\\ref{fn:#1}}"
+                               || name == "\\providecommand\\enuref[1]{\\ref{enu:#1}}"
+                               || name == "\\providecommand\\eqref[1]{\\ref{eq:#1}}"
+                               || name == "\\providecommand\\lemref[1]{\\ref{lem:#1}}"
+                               || name == "\\providecommand\\thmref[1]{\\ref{thm:#1}}"
+                               || name == "\\providecommand\\corref[1]{\\ref{cor:#1}}"
+                               || name == "\\providecommand\\propref[1]{\\ref{prop:#1}}"))
+                               p.skip_spaces();
+                       else {
+                               ostringstream ss;
+                               ss << '\\' << t.cs();
+                               ss << '{' << name << '}';
+                               h_preamble << ss.str();
+                       }
                }
 
                else if (t.cs() == "newcommand" || t.cs() == "newcommandx"
@@ -1299,26 +1605,44 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        string const opt1 = p.getFullOpt();
                        string const opt2 = p.getFullOpt();
                        string const body = p.verbatim_item();
+                       // store the in_lyx_preamble setting
+                       bool const was_in_lyx_preamble = in_lyx_preamble;
                        // font settings
                        if (name == "\\rmdefault")
-                               if (is_known(body, known_roman_fonts))
-                                       h_font_roman = body;
+                               if (is_known(body, known_roman_fonts)) {
+                                       h_font_roman[0] = body;
+                                       p.skip_spaces();
+                                       in_lyx_preamble = true;
+                               }
                        if (name == "\\sfdefault")
-                               if (is_known(body, known_sans_fonts))
-                                       h_font_sans = body;
+                               if (is_known(body, known_sans_fonts)) {
+                                       h_font_sans[0] = body;
+                                       p.skip_spaces();
+                                       in_lyx_preamble = true;
+                               }
                        if (name == "\\ttdefault")
-                               if (is_known(body, known_typewriter_fonts))
-                                       h_font_typewriter = body;
+                               if (is_known(body, known_typewriter_fonts)) {
+                                       h_font_typewriter[0] = body;
+                                       p.skip_spaces();
+                                       in_lyx_preamble = true;
+                               }
                        if (name == "\\familydefault") {
                                string family = body;
                                // remove leading "\"
                                h_font_default_family = family.erase(0,1);
+                               p.skip_spaces();
+                               in_lyx_preamble = true;
                        }
 
-                       // remove the lyxdot definition that is re-added by LyX
+                       // remove LyX-specific definitions that are re-added by LyX
                        // if necessary
-                       if (name == "\\lyxdot")
+                       // \lyxline is an ancient command that is converted by tex2lyx into
+                       // a \rule therefore remove its preamble code
+                       if (name == "\\lyxdot" || name == "\\lyxarrow"
+                           || name == "\\lyxline" || name == "\\LyX") {
+                               p.skip_spaces();
                                in_lyx_preamble = true;
+                       }
 
                        // Add the command to the known commands
                        add_known_command(name, opt1, !opt2.empty(), from_utf8(body));
@@ -1338,6 +1662,8 @@ void Preamble::parse(Parser & p, string const & forceclass,
                                    << opts << "{" << body << "}";
 */
                        }
+                       // restore the in_lyx_preamble setting
+                       in_lyx_preamble = was_in_lyx_preamble;
                }
 
                else if (t.cs() == "documentclass") {
@@ -1355,6 +1681,12 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        handle_opt(opts, known_languages, h_language);
                        delete_opt(opts, known_languages);
 
+                       // math indentation
+                       if ((it = find(opts.begin(), opts.end(), "fleqn"))
+                                != opts.end()) {
+                               h_is_mathindent = "1";
+                               opts.erase(it);
+                       }
                        // paper orientation
                        if ((it = find(opts.begin(), opts.end(), "landscape")) != opts.end()) {
                                h_paperorientation = "landscape";
@@ -1392,6 +1724,7 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        // FIXME This does not work for classes that have a
                        //       different name in LyX than in LaTeX
                        h_textclass = p.getArg('{', '}');
+                       p.skip_spaces();
                }
 
                else if (t.cs() == "usepackage") {
@@ -1403,13 +1736,22 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        vector<string>::const_iterator end = vecnames.end();
                        for (; it != end; ++it)
                                handle_package(p, trimSpaceAndEol(*it), options,
-                                              in_lyx_preamble);
+                                              in_lyx_preamble, detectEncoding);
                }
 
                else if (t.cs() == "inputencoding") {
                        string const encoding = p.getArg('{','}');
-                       h_inputencoding = encoding;
-                       p.setEncoding(encoding);
+                       Encoding const * const enc = encodings.fromLaTeXName(
+                               encoding, Encoding::inputenc, true);
+                       if (!enc) {
+                               if (!detectEncoding)
+                                       cerr << "Unknown encoding " << encoding
+                                            << ". Ignoring." << std::endl;
+                       } else {
+                               if (!enc->unsafe())
+                                       h_inputencoding = enc->name();
+                               p.setEncoding(enc->iconvName());
+                       }
                }
 
                else if (t.cs() == "newenvironment") {
@@ -1428,6 +1770,22 @@ void Preamble::parse(Parser & p, string const & forceclass,
 
                }
 
+               else if (t.cs() == "newtheorem") {
+                       string const name = p.getArg('{', '}');
+                       string const opt1 = p.getFullOpt();
+                       string const opt2 = p.getFullOpt();
+                       string const body = p.verbatim_item();
+                       string const opt3 = p.getFullOpt();
+
+                       string const complete = "\\newtheorem{" + name + '}' +
+                                         opt1 + opt2 + '{' + body + '}' + opt3;
+
+                       add_known_theorem(name, opt1, !opt2.empty(), from_utf8(complete));
+
+                       if (!in_lyx_preamble)
+                               h_preamble << complete;
+               }
+
                else if (t.cs() == "def") {
                        string name = p.get_token().cs();
                        // In fact, name may be more than the name:
@@ -1450,7 +1808,7 @@ void Preamble::parse(Parser & p, string const & forceclass,
                                istringstream is(string(opts, 1));
                                is >> nargs;
                        }
-                       special_columns[name[0]] = nargs;
+                       special_columns_[name[0]] = nargs;
                        h_preamble << "\\newcolumntype{" << name << "}";
                        if (nargs)
                                h_preamble << "[" << nargs << "]";
@@ -1485,7 +1843,9 @@ void Preamble::parse(Parser & p, string const & forceclass,
                                else if (content == "\\bigskipamount")
                                        h_defskip = "bigskip";
                                else
-                                       h_defskip = content;
+                                       h_defskip = translate_len(content);
+                       } else if (name == "\\mathindent") {
+                               h_mathindentation = translate_len(content);
                        } else
                                h_preamble << "\\setlength{" << name << "}{" << content << "}";
                }
@@ -1632,8 +1992,6 @@ void Preamble::parse(Parser & p, string const & forceclass,
        // Force textclass if the user wanted it
        if (!forceclass.empty())
                h_textclass = forceclass;
-       if (noweb_mode && !prefixIs(h_textclass, "literate-"))
-               h_textclass.insert(0, "literate-");
        tc.setName(h_textclass);
        if (!tc.load()) {
                cerr << "Error: Could not read layout file for textclass \"" << h_textclass << "\"." << endl;
@@ -1644,6 +2002,43 @@ void Preamble::parse(Parser & p, string const & forceclass,
                ss << tc.sides();
                h_papersides = ss.str();
        }
+
+       // If the CJK package is used we cannot set the document language from
+       // the babel options. Instead, we guess which language is used most
+       // and set this one.
+       default_language = h_language;
+       if (is_full_document &&
+           (auto_packages.find("CJK") != auto_packages.end() ||
+            auto_packages.find("CJKutf8") != auto_packages.end())) {
+               p.pushPosition();
+               h_language = guessLanguage(p, default_language);
+               p.popPosition();
+               if (explicit_babel && h_language != default_language) {
+                       // We set the document language to a CJK language,
+                       // but babel is explicitly called in the user preamble
+                       // without options. LyX will not add the default
+                       // language to the document options if it is either
+                       // english, or no text is set as default language.
+                       // Therefore we need to add a language option explicitly.
+                       // FIXME: It would be better to remove all babel calls
+                       //        from the user preamble, but this is difficult
+                       //        without re-introducing bug 7861.
+                       if (h_options.empty())
+                               h_options = lyx2babel(default_language);
+                       else
+                               h_options += ',' + lyx2babel(default_language);
+               }
+       }
+}
+
+
+string Preamble::parseEncoding(Parser & p, string const & forceclass)
+{
+       TeX2LyXDocClass dummy;
+       parse(p, forceclass, true, dummy);
+       if (h_inputencoding != "auto" && h_inputencoding != "default")
+               return h_inputencoding;
+       return "";
 }
 
 
@@ -1656,6 +2051,15 @@ string babel2lyx(string const & language)
 }
 
 
+string lyx2babel(string const & language)
+{
+       char const * const * where = is_known(language, known_coded_languages);
+       if (where)
+               return known_languages[where - known_coded_languages];
+       return language;
+}
+
+
 string Preamble::polyglossia2lyx(string const & language)
 {
        char const * const * where = is_known(language, polyglossia_languages);