]> git.lyx.org Git - lyx.git/blobdiff - src/tex2lyx/Preamble.cpp
Move stuff around
[lyx.git] / src / tex2lyx / Preamble.cpp
index 6261fa54a271822ad59434922ffacf2c719a72ad..82135a8621d7db71741c74ebbe49ff93d24b192c 100644 (file)
@@ -16,6 +16,7 @@
 #include "Preamble.h"
 #include "tex2lyx.h"
 
+#include "Encoding.h"
 #include "LayoutFile.h"
 #include "Layout.h"
 #include "Lexer.h"
@@ -44,28 +45,23 @@ Preamble preamble;
 
 namespace {
 
-//add this to known_languages when updating to lyxformat 266:
-// "armenian" (needs special handling since not supported by standard babel)
-//add these to known_languages when updating to lyxformat 268:
-//"chinese-simplified", "chinese-traditional", "japanese", "korean"
-// Both changes require first that support for non-babel languages (CJK,
-// armtex) is added.
+// CJK languages are handled in text.cpp, polyglossia languages are listed
+// further down.
 /**
  * known babel language names (including synonyms)
  * not in standard babel: arabic, arabtex, armenian, belarusian, serbian-latin, thai
- * not yet supported by LyX: kurmanji
  * please keep this in sync with known_coded_languages line by line!
  */
 const char * const known_languages[] = {"acadian", "afrikaans", "albanian",
-"american", "arabic", "arabtex", "austrian", "bahasa", "bahasai", "bahasam",
-"basque", "belarusian", "brazil", "brazilian", "breton", "british", "bulgarian",
-"canadian", "canadien", "catalan", "croatian", "czech", "danish", "dutch",
-"english", "esperanto", "estonian", "farsi", "finnish", "francais", "french",
-"frenchb", "frenchle", "frenchpro", "galician", "german", "germanb", "greek",
-"hebrew", "hungarian", "icelandic", "indon", "indonesian", "interlingua",
-"irish", "italian", "kazakh", "latin", "latvian", "lithuanian", "lowersorbian",
-"lsorbian", "magyar", "malay", "meyalu", "mongolian", "naustrian", "newzealand",
-"ngerman", "ngermanb", "norsk", "nynorsk", "polutonikogreek", "polish",
+"american", "arabic", "arabtex", "australian", "austrian", "bahasa", "bahasai",
+"bahasam", "basque", "belarusian", "brazil", "brazilian", "breton", "british",
+"bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
+"dutch", "english", "esperanto", "estonian", "farsi", "finnish", "francais",
+"french", "frenchb", "frenchle", "frenchpro", "galician", "german", "germanb",
+"greek", "hebrew", "hungarian", "icelandic", "indon", "indonesian", "interlingua",
+"irish", "italian", "japanese", "kazakh", "kurmanji", "latin", "latvian", "lithuanian",
+"lowersorbian", "lsorbian", "magyar", "malay", "meyalu", "mongolian", "naustrian",
+"newzealand", "ngerman", "ngermanb", "norsk", "nynorsk", "polutonikogreek", "polish",
 "portuges", "portuguese", "romanian", "russian", "russianb", "samin",
 "scottish", "serbian", "serbian-latin", "slovak", "slovene", "spanish",
 "swedish", "thai", "turkish", "turkmen", "ukraineb", "ukrainian",
@@ -77,25 +73,29 @@ const char * const known_languages[] = {"acadian", "afrikaans", "albanian",
  * please keep this in sync with known_languages line by line!
  */
 const char * const known_coded_languages[] = {"french", "afrikaans", "albanian",
-"american", "arabic_arabi", "arabic_arabtex", "austrian", "bahasa", "bahasa", "bahasam",
-"basque", "belarusian", "brazilian", "brazilian", "breton", "british", "bulgarian",
-"canadian", "canadien", "catalan", "croatian", "czech", "danish", "dutch",
-"english", "esperanto", "estonian", "farsi", "finnish", "french", "french",
-"french", "french", "french", "galician", "german", "german", "greek",
-"hebrew", "magyar", "icelandic", "bahasa", "bahasa", "interlingua",
-"irish", "italian", "kazakh", "latin", "latvian", "lithuanian", "lowersorbian",
-"lowersorbian", "magyar", "bahasam", "bahasam", "mongolian", "naustrian", "english",
-"ngerman", "ngerman", "norsk", "nynorsk", "polutonikogreek", "polish",
+"american", "arabic_arabi", "arabic_arabtex", "australian", "austrian", "bahasa", "bahasa",
+"bahasam", "basque", "belarusian", "brazilian", "brazilian", "breton", "british",
+"bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
+"dutch", "english", "esperanto", "estonian", "farsi", "finnish", "french",
+"french", "french", "french", "french", "galician", "german", "german",
+"greek", "hebrew", "magyar", "icelandic", "bahasa", "bahasa", "interlingua",
+"irish", "italian", "japanese", "kazakh", "kurmanji", "latin", "latvian", "lithuanian",
+"lowersorbian", "lowersorbian", "magyar", "bahasam", "bahasam", "mongolian", "naustrian",
+"newzealand", "ngerman", "ngerman", "norsk", "nynorsk", "polutonikogreek", "polish",
 "portuguese", "portuguese", "romanian", "russian", "russian", "samin",
 "scottish", "serbian", "serbian-latin", "slovak", "slovene", "spanish",
 "swedish", "thai", "turkish", "turkmen", "ukrainian", "ukrainian",
 "uppersorbian", "uppersorbian", "english", "english", "vietnamese", "welsh",
 0};
 
+/// languages with danish quotes (.lyx names)
+const char * const known_danish_quotes_languages[] = {"danish", 0};
+
 /// languages with english quotes (.lyx names)
-const char * const known_english_quotes_languages[] = {"american", "bahasa",
-"bahasam", "brazilian", "canadian", "chinese-simplified", "english",
-"esperanto", "hebrew", "irish", "korean", "portuguese", "scottish", "thai", 0};
+const char * const known_english_quotes_languages[] = {"american", "australian",
+"bahasa", "bahasam", "brazilian", "canadian", "chinese-simplified", "english",
+"esperanto", "hebrew", "irish", "korean", "newzealand", "portuguese", "scottish",
+"thai", 0};
 
 /// languages with french quotes (.lyx names)
 const char * const known_french_quotes_languages[] = {"albanian",
@@ -124,15 +124,21 @@ const char * const known_old_language_packages[] = {"french", "frenchle",
 char const * const known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
 
 const char * const known_roman_fonts[] = { "ae", "beraserif", "bookman",
-"ccfonts", "chancery", "charter", "cmr", "fourier", "lmodern", "mathpazo",
-"mathptmx", "newcent", "utopia", 0};
+"ccfonts", "chancery", "charter", "cmr", "fourier", "garamondx", "libertine",
+"libertine-type1", "lmodern", "mathdesign", "mathpazo", "mathptmx", "newcent",
+"tgbonum", "tgchorus", "tgpagella", "tgschola", "tgtermes", "utopia", 0};
+
+const char * const known_sans_fonts[] = { "avant", "berasans", "biolinum-type1",
+"cmbr", "cmss", "helvet", "kurier", "kurierl", "lmss", "tgadventor", "tgheros", 0};
 
-const char * const known_sans_fonts[] = { "avant", "berasans", "cmbr", "cmss",
-"helvet", "lmss", 0};
+const char * const known_kurier_fonts[] = { "kurier", "kurierl", "kurier-condensed",
+"kurier-light-condensed", 0};
 
 const char * const known_typewriter_fonts[] = { "beramono", "cmtl", "cmtt",
 "courier", "lmtt", "luximono", "fourier", "lmodern", "mathpazo", "mathptmx",
-"newcent", 0};
+"newcent", "tgcursor", "txtt", 0};
+
+const char * const known_math_fonts[] = { "eulervm", "newtxmath", 0};
 
 const char * const known_paper_sizes[] = { "a0paper", "b0paper", "c0paper",
 "a1paper", "b1paper", "c1paper", "a2paper", "b2paper", "c2paper", "a3paper",
@@ -143,7 +149,7 @@ const char * const known_paper_sizes[] = { "a0paper", "b0paper", "c0paper",
 const char * const known_class_paper_sizes[] = { "a4paper", "a5paper",
 "executivepaper", "legalpaper", "letterpaper", 0};
 
-const char * const known_paper_margins[] = { "lmargin", "tmargin", "rmargin", 
+const char * const known_paper_margins[] = { "lmargin", "tmargin", "rmargin",
 "bmargin", "headheight", "headsep", "footskip", "columnsep", 0};
 
 const char * const known_coded_paper_margins[] = { "leftmargin", "topmargin",
@@ -166,10 +172,19 @@ const char * const known_if_3arg_commands[] = {"@ifundefined", "IfFileExists",
 0};
 
 /// packages that work only in xetex
+/// polyglossia is handled separately
 const char * const known_xetex_packages[] = {"arabxetex", "fixlatvian",
-"fontbook", "fontwrap", "mathspec", "philokalia", "polyglossia", "unisugar",
+"fontbook", "fontwrap", "mathspec", "philokalia", "unisugar",
 "xeCJK", "xecolor", "xecyr", "xeindex", "xepersian", "xunicode", 0};
 
+/// packages that are automatically skipped if loaded by LyX
+const char * const known_lyx_packages[] = {"amsbsy", "amsmath", "amssymb",
+"amstext", "amsthm", "array", "babel", "booktabs", "calc", "CJK", "color", "float",
+"fontspec", "graphicx", "hhline", "ifthen", "longtable", "makeidx", "multirow",
+"nomencl", "pdfpages", "rotating", "rotfloat", "splitidx", "setspace",
+"subscript", "textcomp", "ulem", "url", "varioref", "verbatim", "wrapfig",
+"xunicode", 0};
+
 // codes used to remove packages that are loaded automatically by LyX.
 // Syntax: package_beg_sep<name>package_mid_sep<package loading code>package_end_sep
 const char package_beg_sep = '\001';
@@ -276,6 +291,42 @@ string process_keyval_opt(vector<string> & options, string name)
 } // anonymous namespace
 
 
+/**
+ * known polyglossia language names (including variants)
+ */
+const char * const Preamble::polyglossia_languages[] = {
+"albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
+"nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
+"armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
+"irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
+"bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
+"samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
+"brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
+"lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
+"welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
+"marathi", "spanish",
+"american", "ancient", "australian", "british", "monotonic", "newzealand",
+"polytonic", 0};
+
+/**
+ * the same as polyglossia_languages with .lyx names
+ * please keep this in sync with polyglossia_languages line by line!
+ */
+const char * const Preamble::coded_polyglossia_languages[] = {
+"albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
+"nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
+"armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
+"irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
+"bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
+"samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
+"brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
+"lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
+"welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
+"marathi", "spanish",
+"american", "ancientgreek", "australian", "british", "greek", "newzealand",
+"polutonikogreek", 0};
+
+
 bool Preamble::indentParagraphs() const
 {
        return h_paragraph_separation == "indent";
@@ -332,7 +383,7 @@ Author const & Preamble::getAuthor(std::string const & name) const
 {
        Author author(from_utf8(name), empty_docstring());
        for (AuthorList::Authors::const_iterator it = authors_.begin();
-            it != authors_.end(); it++)
+            it != authors_.end(); ++it)
                if (*it == author)
                        return *it;
        static Author const dummy;
@@ -361,18 +412,18 @@ void Preamble::add_package(string const & name, vector<string> & options)
 
 namespace {
 
-// Given is a string like "scaled=0.9", return 0.9 * 100
-string const scale_as_percentage(string const & scale)
+// Given is a string like "scaled=0.9" or "Scale=0.9", return 0.9 * 100
+bool scale_as_percentage(string const & scale, string & percentage)
 {
        string::size_type pos = scale.find('=');
        if (pos != string::npos) {
                string value = scale.substr(pos + 1);
-               if (isStrDbl(value))
-                       return convert<string>(100 * convert<double>(value));
+               if (isStrDbl(value)) {
+                       percentage = convert<string>(100 * convert<double>(value));
+                       return true;
+               }
        }
-       // If the input string didn't match our expectations.
-       // return the default value "100"
-       return "100";
+       return false;
 }
 
 
@@ -388,11 +439,16 @@ string remove_braces(string const & value)
 } // anonymous namespace
 
 
-Preamble::Preamble() : one_language(true)
+Preamble::Preamble() : one_language(true), title_layout_found(false),
+       h_font_cjk_set(false)
 {
        //h_backgroundcolor;
        //h_boxbgcolor;
+       h_biblio_style            = "plain";
+       h_bibtex_command          = "default";
        h_cite_engine             = "basic";
+       h_cite_engine_type        = "numerical";
+       h_color                   = "#008000";
        h_defskip                 = "medskip";
        //h_float_placement;
        //h_fontcolor;
@@ -400,23 +456,33 @@ Preamble::Preamble() : one_language(true)
        h_font_roman              = "default";
        h_font_sans               = "default";
        h_font_typewriter         = "default";
+       h_font_math               = "auto";
        h_font_default_family     = "default";
+       h_use_non_tex_fonts       = "false";
        h_font_sc                 = "false";
        h_font_osf                = "false";
        h_font_sf_scale           = "100";
        h_font_tt_scale           = "100";
+       //h_font_cjk
        h_graphics                = "default";
+       h_default_output_format   = "default";
        h_html_be_strict          = "false";
        h_html_css_as_file        = "0";
        h_html_math_output        = "0";
+       h_index                   = "Index";
+       h_index_command           = "default";
        h_inputencoding           = "auto";
+       h_justification           = "true";
        h_language                = "english";
        h_language_package        = "none";
        //h_listings_params;
+       h_maintain_unincluded_children = "false";
        //h_margins;
        //h_notefontcolor;
        //h_options;
        h_output_changes          = "false";
+       h_output_sync             = "0";
+       //h_output_sync_macro
        h_papercolumns            = "1";
        h_paperfontsize           = "default";
        h_paperorientation        = "portrait";
@@ -442,21 +508,27 @@ Preamble::Preamble() : one_language(true)
        //h_pdf_quoted_options;
        h_quotes_language         = "english";
        h_secnumdepth             = "3";
+       h_shortcut                = "idx";
        h_spacing                 = "single";
        h_suppress_date           = "false";
        h_textclass               = "article";
        h_tocdepth                = "3";
        h_tracking_changes        = "false";
        h_use_bibtopic            = "false";
+       h_use_indices             = "false";
        h_use_geometry            = "false";
-       h_use_amsmath             = "1";
        h_use_default_options     = "false";
-       h_use_esint               = "1";
-       h_use_hyperref            = "0";
-       h_use_mhchem              = "0";
-       h_use_mathdots            = "0";
+       h_use_hyperref            = "false";
        h_use_refstyle            = "0";
-       h_use_undertilde          = "0";
+       h_use_packages["amsmath"]    = "1";
+       h_use_packages["amssymb"]    = "0";
+       h_use_packages["esint"]      = "1";
+       h_use_packages["mhchem"]     = "0";
+       h_use_packages["mathdots"]   = "0";
+       h_use_packages["mathtools"]  = "0";
+       h_use_packages["stackrel"]   = "0";
+       h_use_packages["stmaryrd"]   = "0";
+       h_use_packages["undertilde"] = "0";
 }
 
 
@@ -464,7 +536,7 @@ void Preamble::handle_hyperref(vector<string> & options)
 {
        // FIXME swallow inputencoding changes that might surround the
        //       hyperref setup if it was written by LyX
-       h_use_hyperref = "1";
+       h_use_hyperref = "true";
        // swallow "unicode=true", since LyX does always write that
        vector<string>::iterator it =
                find(options.begin(), options.end(), "unicode=true");
@@ -579,16 +651,19 @@ void Preamble::handle_package(Parser &p, string const & name,
 {
        vector<string> options = split_options(opts);
        add_package(name, options);
-       string scale;
+       char const * const * where = 0;
 
-       if (is_known(name, known_xetex_packages))
+       if (is_known(name, known_xetex_packages)) {
                xetex = true;
+               h_use_non_tex_fonts = "true";
+               registerAutomaticallyLoadedPackage("fontspec");
+               if (h_inputencoding == "auto")
+                       p.setEncoding("UTF-8");
+       }
 
        // roman fonts
-       if (is_known(name, known_roman_fonts)) {
+       if (is_known(name, known_roman_fonts))
                h_font_roman = name;
-               p.skip_spaces();
-       }
 
        if (name == "fourier") {
                h_font_roman = "utopia";
@@ -597,30 +672,72 @@ void Preamble::handle_package(Parser &p, string const & name,
                        h_font_sc = "true";
        }
 
-       if (name == "mathpazo")
+       if (name == "garamondx") {
+               h_font_roman = "garamondx";
+               if (opts == "osfI")
+                       h_font_osf = "true";
+       }
+
+       if (name == "libertine") {
+               h_font_roman = "libertine";
+               // this automatically invokes biolinum
+               h_font_sans = "biolinum";
+               if (opts == "osf")
+                       h_font_osf = "true";
+               else if (opts == "lining")
+                       h_font_osf = "false";
+       }
+
+       if (name == "libertine-type1") {
+               h_font_roman = "libertine";
+               // NOTE: contrary to libertine.sty, libertine-type1
+               // does not automatically invoke biolinum
+               if (opts == "lining")
+                       h_font_osf = "false";
+               else if (opts == "osf")
+                       h_font_osf = "true";
+       }
+       
+       if (name == "mathdesign") {
+               if (opts.find("charter") != string::npos)
+                       h_font_roman = "md-charter";
+               if (opts.find("garamond") != string::npos)
+                       h_font_roman = "md-garamond";
+               if (opts.find("utopia") != string::npos)
+                       h_font_roman = "md-utopia";
+               if (opts.find("expert") != string::npos) {
+                       h_font_sc = "true";
+                       h_font_osf = "true";
+               }
+       }
+
+       else if (name == "mathpazo")
                h_font_roman = "palatino";
 
-       if (name == "mathptmx")
+       else if (name == "mathptmx")
                h_font_roman = "times";
 
        // sansserif fonts
        if (is_known(name, known_sans_fonts)) {
                h_font_sans = name;
-               if (!opts.empty()) {
-                       scale = opts;
-                       h_font_sf_scale = scale_as_percentage(scale);
+               if (options.size() == 1) {
+                       if (scale_as_percentage(opts, h_font_sf_scale))
+                               options.clear();
                }
        }
 
+       if (name == "biolinum-type1")
+               h_font_sans = "biolinum";
+
        // typewriter fonts
        if (is_known(name, known_typewriter_fonts)) {
                // fourier can be set as roman font _only_
                // fourier as typewriter is handled in handling of \ttdefault
                if (name != "fourier") {
                        h_font_typewriter = name;
-                       if (!opts.empty()) {
-                               scale = opts;
-                               h_font_tt_scale = scale_as_percentage(scale);
+                       if (options.size() == 1) {
+                               if (scale_as_percentage(opts, h_font_tt_scale))
+                                       options.clear();
                        }
                }
        }
@@ -629,26 +746,35 @@ void Preamble::handle_package(Parser &p, string const & name,
        if (name == "eco")
                h_font_osf = "true";
 
+       // math fonts
+       if (is_known(name, known_math_fonts))
+               h_font_math = name;
+
+       if (name == "newtxmath") {
+               if (opts.empty())
+                       h_font_math = "newtxmath";
+               else if (opts == "garamondx")
+                       h_font_math = "garamondx-ntxm";
+               else if (opts == "libertine")
+                       h_font_math = "libertine-ntxm";
+               else if (opts == "minion")
+                       h_font_math = "minion-ntxm";
+       }
+
+       if (name == "refstyle")
+               h_use_refstyle = "1";
+
        // after the detection and handling of special cases, we can remove the
        // fonts, otherwise they would appear in the preamble, see bug #7856
        if (is_known(name, known_roman_fonts) || is_known(name, known_sans_fonts)
-               ||      is_known(name, known_typewriter_fonts))
+               ||      is_known(name, known_typewriter_fonts) || is_known(name, known_math_fonts))
                ;
 
-       else if (name == "amsmath" || name == "amssymb")
-               h_use_amsmath = "2";
-
-       else if (name == "esint")
-               h_use_esint = "2";
-
-       else if (name == "mhchem")
-               h_use_mhchem = "2";
-
-       else if (name == "mathdots")
-               h_use_mathdots = "2";
-
-       else if (name == "undertilde")
-               h_use_undertilde = "2";
+       else if (name == "amsmath" || name == "amssymb" ||
+                name == "esint" || name == "mhchem" || name == "mathdots" ||
+                name == "mathtools" || name == "stackrel" ||
+                name == "stmaryrd" || name == "undertilde")
+               h_use_packages[name] = "2";
 
        else if (name == "babel") {
                h_language_package = "default";
@@ -665,17 +791,61 @@ void Preamble::handle_package(Parser &p, string const & name,
                        // call as document language. If there is no such language option, the
                        // last language in the documentclass options is used.
                        handle_opt(options, known_languages, h_language);
-                       // If babel is called with options, LyX puts them by default into the
-                       // document class options. This works for most languages, except
-                       // for Latvian, Lithuanian, Mongolian, Turkmen and Vietnamese and
-                       // perhaps in future others.
-                       // Therefore keep the babel call as it is as the user might have
-                       // reasons for it.
-                       h_preamble << "\\usepackage[" << opts << "]{babel}\n";
+                       // translate the babel name to a LyX name
+                       h_language = babel2lyx(h_language);
+                       if (h_language == "japanese") {
+                               // For Japanese, the encoding isn't indicated in the source
+                               // file, and there's really not much we can do. We could
+                               // 1) offer a list of possible encodings to choose from, or
+                               // 2) determine the encoding of the file by inspecting it.
+                               // For the time being, we leave the encoding alone so that
+                               // we don't get iconv errors when making a wrong guess, and
+                               // we will output a note at the top of the document
+                               // explaining what to do.
+                               Encoding const * const enc = encodings.fromIconvName(
+                                       p.getEncoding(), Encoding::japanese, false);
+                               if (enc)
+                                       h_inputencoding = enc->name();
+                               is_nonCJKJapanese = true;
+                               // in this case babel can be removed from the preamble
+                               registerAutomaticallyLoadedPackage("babel");
+                       } else {
+                               // If babel is called with options, LyX puts them by default into the
+                               // document class options. This works for most languages, except
+                               // for Latvian, Lithuanian, Mongolian, Turkmen and Vietnamese and
+                               // perhaps in future others.
+                               // Therefore keep the babel call as it is as the user might have
+                               // reasons for it.
+                               h_preamble << "\\usepackage[" << opts << "]{babel}\n";
+                       }
                        delete_opt(options, known_languages);
                }
                else
-                       h_preamble << "\\usepackage{babel}\n";                                          
+                       h_preamble << "\\usepackage{babel}\n";
+       }
+
+       else if (name == "polyglossia") {
+               h_language_package = "default";
+               h_default_output_format = "pdf4";
+               h_use_non_tex_fonts = "true";
+               xetex = true;
+               registerAutomaticallyLoadedPackage("xunicode");
+               if (h_inputencoding == "auto")
+                       p.setEncoding("UTF-8");
+       }
+
+       else if (name == "CJK") {
+               // set the encoding to "auto" because it might be set to "default" by the babel handling
+               // and this would not be correct for CJK
+               if (h_inputencoding == "default")
+                       h_inputencoding = "auto";
+               registerAutomaticallyLoadedPackage("CJK");
+       }
+
+       else if (name == "CJKutf8") {
+               h_inputencoding = "UTF8";
+               p.setEncoding("UTF-8");
+               registerAutomaticallyLoadedPackage("CJKutf8");
        }
 
        else if (name == "fontenc") {
@@ -698,68 +868,63 @@ void Preamble::handle_package(Parser &p, string const & name,
                if (opts.find(",") == string::npos && one_language == true)
                        h_inputencoding = opts;
                if (!options.empty())
-                       p.setEncoding(options.back());
+                       p.setEncoding(options.back(), Encoding::inputenc);
                options.clear();
        }
 
+       else if (name == "srcltx") {
+               h_output_sync = "1";
+               if (!opts.empty()) {
+                       h_output_sync_macro = "\\usepackage[" + opts + "]{srcltx}";
+                       options.clear();
+               } else
+                       h_output_sync_macro = "\\usepackage{srcltx}";
+       }
+
        else if (is_known(name, known_old_language_packages)) {
                // known language packages from the times before babel
                // if they are found and not also babel, they will be used as
-               // cutom language package
+               // custom language package
                h_language_package = "\\usepackage{" + name + "}";
        }
 
-       else if (name == "makeidx")
-               ; // ignore this
-
        else if (name == "prettyref")
-               ; // ignore this
-
-       else if (name == "varioref")
-               ; // ignore this
-
-       else if (name == "verbatim")
-               ; // ignore this
-
-       else if (name == "nomencl")
-               ; // ignore this
+               ; // ignore this FIXME: Use the package separator mechanism instead
 
-       else if (name == "textcomp")
-               ; // ignore this
-
-       else if (name == "url")
-               ; // ignore this
+       else if (name == "lyxskak") {
+               // ignore this and its options
+               const char * const o[] = {"ps", "mover", 0};
+               delete_opt(options, o);
+       }
 
-       else if (name == "color" || name == "subscript" || name == "ulem") {
-               if (!in_lyx_preamble)
+       else if (is_known(name, known_lyx_packages) && options.empty()) {
+               if (name == "splitidx")
+                       h_use_indices = "true";
+               if (!in_lyx_preamble) {
                        h_preamble << package_beg_sep << name
                                   << package_mid_sep << "\\usepackage{"
-                                  << name << '}' << package_end_sep;
+                                  << name << '}';
+                       if (p.next_token().cat() == catNewline ||
+                           (p.next_token().cat() == catSpace &&
+                            p.next_next_token().cat() == catNewline))
+                               h_preamble << '\n';
+                       h_preamble << package_end_sep;
+               }
        }
 
-       else if (name == "graphicx")
-               ; // ignore this
-
-       else if (name == "setspace")
-               ; // ignore this
-
        else if (name == "geometry")
                handle_geometry(options);
 
-       else if (name == "rotfloat")
-               ; // ignore this
-
-       else if (name == "wrapfig")
-               ; // ignore this
-
        else if (name == "subfig")
-               ; // ignore this
+               ; // ignore this FIXME: Use the package separator mechanism instead
 
-       else if (is_known(name, known_languages))
-               h_language = name;
+       else if ((where = is_known(name, known_languages)))
+               h_language = known_coded_languages[where - known_languages];
 
        else if (name == "natbib") {
-               h_cite_engine = "natbib_authoryear";
+               h_biblio_style = "plainnat";
+               h_cite_engine = "natbib";
+               h_cite_engine_type = "authoryear";
                vector<string>::iterator it =
                        find(options.begin(), options.end(), "authoryear");
                if (it != options.end())
@@ -767,26 +932,33 @@ void Preamble::handle_package(Parser &p, string const & name,
                else {
                        it = find(options.begin(), options.end(), "numbers");
                        if (it != options.end()) {
-                               h_cite_engine = "natbib_numerical";
+                               h_cite_engine_type = "numerical";
                                options.erase(it);
                        }
                }
        }
 
-       else if (name == "jurabib")
+       else if (name == "jurabib") {
+               h_biblio_style = "jurabib";
                h_cite_engine = "jurabib";
+               h_cite_engine_type = "authoryear";
+       }
 
        else if (name == "hyperref")
                handle_hyperref(options);
 
        else if (!in_lyx_preamble) {
                if (options.empty())
-                       h_preamble << "\\usepackage{" << name << "}";
+                       h_preamble << "\\usepackage{" << name << '}';
                else {
-                       h_preamble << "\\usepackage[" << opts << "]{" 
-                                  << name << "}";
+                       h_preamble << "\\usepackage[" << opts << "]{"
+                                  << name << '}';
                        options.clear();
                }
+               if (p.next_token().cat() == catNewline ||
+                   (p.next_token().cat() == catSpace &&
+                    p.next_next_token().cat() == catNewline))
+                       h_preamble << '\n';
        }
 
        // We need to do something with the options...
@@ -816,11 +988,8 @@ void Preamble::handle_if(Parser & p, bool in_lyx_preamble)
 }
 
 
-bool Preamble::writeLyXHeader(ostream & os)
+bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
 {
-       // translate from babel to LyX names
-       h_language = babel2lyx(h_language);
-
        // set the quote language
        // LyX only knows the following quotes languages:
        // english, swedish, german, polish, french and danish
@@ -830,7 +999,7 @@ bool Preamble::writeLyXHeader(ostream & os)
        // http://en.wikipedia.org/wiki/Quotation_mark,_non-English_usage
        // (quotes for kazakh and interlingua are unknown)
        // danish
-       if (h_language == "danish")
+       if (is_known(h_language, known_danish_quotes_languages))
                h_quotes_language = "danish";
        // french
        else if (is_known(h_language, known_french_quotes_languages))
@@ -848,13 +1017,23 @@ bool Preamble::writeLyXHeader(ostream & os)
        else if (is_known(h_language, known_english_quotes_languages))
                h_quotes_language = "english";
 
+       if (contains(h_float_placement, "H"))
+               registerAutomaticallyLoadedPackage("float");
+       if (h_spacing != "single" && h_spacing != "default")
+               registerAutomaticallyLoadedPackage("setspace");
+       if (h_use_packages["amsmath"] == "2") {
+               // amsbsy and amstext are already provided by amsmath
+               registerAutomaticallyLoadedPackage("amsbsy");
+               registerAutomaticallyLoadedPackage("amstext");
+       }
+
        // output the LyX file settings
        os << "#LyX file created by tex2lyx " << PACKAGE_VERSION << "\n"
           << "\\lyxformat " << LYX_FORMAT << '\n'
           << "\\begin_document\n"
           << "\\begin_header\n"
           << "\\textclass " << h_textclass << "\n";
-       string const raw = h_preamble.str();
+       string const raw = subdoc ? empty_string() : h_preamble.str();
        if (!raw.empty()) {
                os << "\\begin_preamble\n";
                for (string::size_type i = 0; i < raw.size(); ++i) {
@@ -885,29 +1064,40 @@ bool Preamble::writeLyXHeader(ostream & os)
                os << "\\begin_modules\n";
                vector<string>::const_iterator const end = used_modules.end();
                vector<string>::const_iterator it = used_modules.begin();
-               for (; it != end; it++)
+               for (; it != end; ++it)
                        os << *it << '\n';
                os << "\\end_modules\n";
        }
-       os << "\\language " << h_language << "\n"
+       os << "\\maintain_unincluded_children " << h_maintain_unincluded_children << "\n"
+          << "\\language " << h_language << "\n"
           << "\\language_package " << h_language_package << "\n"
           << "\\inputencoding " << h_inputencoding << "\n"
           << "\\fontencoding " << h_fontencoding << "\n"
           << "\\font_roman " << h_font_roman << "\n"
           << "\\font_sans " << h_font_sans << "\n"
           << "\\font_typewriter " << h_font_typewriter << "\n"
+          << "\\font_math " << h_font_math << "\n"
           << "\\font_default_family " << h_font_default_family << "\n"
+          << "\\use_non_tex_fonts " << h_use_non_tex_fonts << "\n"
           << "\\font_sc " << h_font_sc << "\n"
           << "\\font_osf " << h_font_osf << "\n"
           << "\\font_sf_scale " << h_font_sf_scale << "\n"
-          << "\\font_tt_scale " << h_font_tt_scale << "\n"
-          << "\\graphics " << h_graphics << "\n";
+          << "\\font_tt_scale " << h_font_tt_scale << '\n';
+       if (!h_font_cjk.empty())
+               os << "\\font_cjk " << h_font_cjk << '\n';
+       os << "\\graphics " << h_graphics << '\n'
+          << "\\default_output_format " << h_default_output_format << "\n"
+          << "\\output_sync " << h_output_sync << "\n";
+       if (h_output_sync == "1")
+               os << "\\output_sync_macro \"" << h_output_sync_macro << "\"\n";
+       os << "\\bibtex_command " << h_bibtex_command << "\n"
+          << "\\index_command " << h_index_command << "\n";
        if (!h_float_placement.empty())
                os << "\\float_placement " << h_float_placement << "\n";
        os << "\\paperfontsize " << h_paperfontsize << "\n"
           << "\\spacing " << h_spacing << "\n"
           << "\\use_hyperref " << h_use_hyperref << '\n';
-       if (h_use_hyperref == "1") {
+       if (h_use_hyperref == "true") {
                if (!h_pdf_title.empty())
                        os << "\\pdf_title \"" << h_pdf_title << "\"\n";
                if (!h_pdf_author.empty())
@@ -931,16 +1121,18 @@ bool Preamble::writeLyXHeader(ostream & os)
                        os << "\\pdf_quoted_options \"" << h_pdf_quoted_options << "\"\n";
        }
        os << "\\papersize " << h_papersize << "\n"
-          << "\\use_geometry " << h_use_geometry << "\n"
-          << "\\use_amsmath " << h_use_amsmath << "\n"
-          << "\\use_esint " << h_use_esint << "\n"
-          << "\\use_mhchem " << h_use_mhchem << "\n"
-          << "\\use_mathdots " << h_use_mathdots << "\n"
-          << "\\use_undertilde " << h_use_undertilde << "\n"
-          << "\\cite_engine " << h_cite_engine << "\n"
+          << "\\use_geometry " << h_use_geometry << '\n';
+       for (map<string, string>::const_iterator it = h_use_packages.begin();
+            it != h_use_packages.end(); ++it)
+               os << "\\use_package " << it->first << ' ' << it->second << '\n';
+       os << "\\cite_engine " << h_cite_engine << '\n'
+          << "\\cite_engine_type " << h_cite_engine_type << '\n'
+          << "\\biblio_style " << h_biblio_style << "\n"
           << "\\use_bibtopic " << h_use_bibtopic << "\n"
+          << "\\use_indices " << h_use_indices << "\n"
           << "\\paperorientation " << h_paperorientation << '\n'
           << "\\suppress_date " << h_suppress_date << '\n'
+          << "\\justification " << h_justification << '\n'
           << "\\use_refstyle " << h_use_refstyle << '\n';
        if (!h_fontcolor.empty())
                os << "\\fontcolor " << h_fontcolor << '\n';
@@ -950,6 +1142,10 @@ bool Preamble::writeLyXHeader(ostream & os)
                os << "\\backgroundcolor " << h_backgroundcolor << '\n';
        if (!h_boxbgcolor.empty())
                os << "\\boxbgcolor " << h_boxbgcolor << '\n';
+       os << "\\index " << h_index << '\n'
+          << "\\shortcut " << h_shortcut << '\n'
+          << "\\color " << h_color << '\n'
+          << "\\end_index\n";
        os << h_margins
           << "\\secnumdepth " << h_secnumdepth << "\n"
           << "\\tocdepth " << h_tocdepth << "\n"
@@ -972,8 +1168,6 @@ bool Preamble::writeLyXHeader(ostream & os)
           << authors_
           << "\\end_header\n\n"
           << "\\begin_body\n";
-       // clear preamble for subdocuments
-       h_preamble.str("");
        return true;
 }
 
@@ -1021,7 +1215,7 @@ void Preamble::parse(Parser & p, string const & forceclass,
                     t.cat() == catParameter))
                        h_preamble << t.cs();
 
-               else if (!in_lyx_preamble && 
+               else if (!in_lyx_preamble &&
                         (t.cat() == catSpace || t.cat() == catNewline))
                        h_preamble << t.asInput();
 
@@ -1034,7 +1228,7 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        // magically switch encoding default if it looks like XeLaTeX
                        static string const magicXeLaTeX =
                                "% This document must be compiled with XeLaTeX ";
-                       if (comment.size() > magicXeLaTeX.size() 
+                       if (comment.size() > magicXeLaTeX.size()
                                  && comment.substr(0, magicXeLaTeX.size()) == magicXeLaTeX
                                  && h_inputencoding == "auto") {
                                cerr << "XeLaTeX comment found, switching to UTF8\n";
@@ -1054,6 +1248,70 @@ void Preamble::parse(Parser & p, string const & forceclass,
                else if (t.cs() == "pagestyle")
                        h_paperpagestyle = p.verbatim_item();
 
+               else if (t.cs() == "setdefaultlanguage") {
+                       xetex = true;
+                       // We don't yet care about non-language variant options
+                       // because LyX doesn't support this yet, see bug #8214
+                       if (p.hasOpt()) {
+                               string langopts = p.getOpt();
+                               // check if the option contains a variant, if yes, extract it
+                               string::size_type pos_var = langopts.find("variant");
+                               string::size_type i = langopts.find(',', pos_var);
+                               string::size_type k = langopts.find('=', pos_var);
+                               if (pos_var != string::npos){
+                                       string variant;
+                                       if (i == string::npos)
+                                               variant = langopts.substr(k + 1, langopts.length() - k - 2);
+                                       else
+                                               variant = langopts.substr(k + 1, i - k - 1);
+                                       h_language = variant;
+                               }
+                               p.verbatim_item();
+                       } else
+                               h_language = p.verbatim_item();
+                       //finally translate the poyglossia name to a LyX name
+                       h_language = polyglossia2lyx(h_language);
+               }
+
+               else if (t.cs() == "setotherlanguage") {
+                       // We don't yet care about the option because LyX doesn't
+                       // support this yet, see bug #8214
+                       p.hasOpt() ? p.getOpt() : string();
+                       p.verbatim_item();
+               }
+
+               else if (t.cs() == "setmainfont") {
+                       // we don't care about the option
+                       p.hasOpt() ? p.getOpt() : string();
+                       h_font_roman = p.getArg('{', '}');
+               }
+
+               else if (t.cs() == "setsansfont" || t.cs() == "setmonofont") {
+                       // LyX currently only supports the scale option
+                       string scale;
+                       if (p.hasOpt()) {
+                               string fontopts = p.getArg('[', ']');
+                               // check if the option contains a scaling, if yes, extract it
+                               string::size_type pos = fontopts.find("Scale");
+                               if (pos != string::npos) {
+                                       string::size_type i = fontopts.find(',', pos);
+                                       if (i == string::npos)
+                                               scale_as_percentage(fontopts.substr(pos + 1), scale);
+                                       else
+                                               scale_as_percentage(fontopts.substr(pos, i - pos), scale);
+                               }
+                       }
+                       if (t.cs() == "setsansfont") {
+                               if (!scale.empty())
+                                       h_font_sf_scale = scale;
+                               h_font_sans = p.getArg('{', '}');
+                       } else {
+                               if (!scale.empty())
+                                       h_font_tt_scale = scale;
+                               h_font_typewriter = p.getArg('{', '}');
+                       }
+               }
+
                else if (t.cs() == "date") {
                        string argument = p.getArg('{', '}');
                        if (argument.empty())
@@ -1104,12 +1362,12 @@ void Preamble::parse(Parser & p, string const & forceclass,
 
                else if (t.cs() == "makeatletter") {
                        // LyX takes care of this
-                       p.setCatCode('@', catLetter);
+                       p.setCatcode('@', catLetter);
                }
 
                else if (t.cs() == "makeatother") {
                        // LyX takes care of this
-                       p.setCatCode('@', catOther);
+                       p.setCatcode('@', catOther);
                }
 
                else if (t.cs() == "newcommand" || t.cs() == "newcommandx"
@@ -1128,20 +1386,47 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        string const opt1 = p.getFullOpt();
                        string const opt2 = p.getFullOpt();
                        string const body = p.verbatim_item();
+                       // store the in_lyx_preamble setting
+                       bool const was_in_lyx_preamble = in_lyx_preamble;
                        // font settings
                        if (name == "\\rmdefault")
-                               if (is_known(body, known_roman_fonts))
+                               if (is_known(body, known_roman_fonts)) {
                                        h_font_roman = body;
+                                       p.skip_spaces();
+                                       in_lyx_preamble = true;
+                               }
                        if (name == "\\sfdefault")
-                               if (is_known(body, known_sans_fonts))
+                               if (is_known(body, known_sans_fonts)) {
                                        h_font_sans = body;
+                                       p.skip_spaces();
+                                       in_lyx_preamble = true;
+                               }
                        if (name == "\\ttdefault")
-                               if (is_known(body, known_typewriter_fonts))
+                               if (is_known(body, known_typewriter_fonts)) {
                                        h_font_typewriter = body;
+                                       p.skip_spaces();
+                                       in_lyx_preamble = true;
+                               }
                        if (name == "\\familydefault") {
                                string family = body;
                                // remove leading "\"
                                h_font_default_family = family.erase(0,1);
+                               p.skip_spaces();
+                               in_lyx_preamble = true;
+                       }
+
+                       if (name == "\\bfdefault")
+                               // LyX re-adds this if a kurier font is used
+                               if (is_known(h_font_sans, known_kurier_fonts) && body == "b") {
+                                       p.skip_spaces();
+                                       in_lyx_preamble = true;
+                               }
+
+                       // remove the lyxdot definition that is re-added by LyX
+                       // if necessary
+                       if (name == "\\lyxdot") {
+                               p.skip_spaces();
+                               in_lyx_preamble = true;
                        }
 
                        // Add the command to the known commands
@@ -1162,6 +1447,24 @@ void Preamble::parse(Parser & p, string const & forceclass,
                                    << opts << "{" << body << "}";
 */
                        }
+                       // restore the in_lyx_preamble setting
+                       in_lyx_preamble = was_in_lyx_preamble;
+               }
+
+               else if (t.cs() == "edef"){
+                       // we only support this for kurier fonts
+                       string const command = p.next_token().asInput();
+                       p.get_token();
+                       if (command == "\\sfdefault") {
+                               p.getArg('{', '}');
+                               if (h_font_sans == "kurier")
+                                       h_font_sans = "kurier-condensed";
+                               if (h_font_sans == "kurierl")
+                                       h_font_sans = "kurier-light-condensed";
+                               p.skip_spaces();
+                       }
+                       else
+                               h_preamble << "\\edef" << command << "{" << p.getArg('{', '}') << "}\n";
                }
 
                else if (t.cs() == "documentclass") {
@@ -1216,6 +1519,7 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        // FIXME This does not work for classes that have a
                        //       different name in LyX than in LaTeX
                        h_textclass = p.getArg('{', '}');
+                       p.skip_spaces();
                }
 
                else if (t.cs() == "usepackage") {
@@ -1226,14 +1530,14 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        vector<string>::const_iterator it  = vecnames.begin();
                        vector<string>::const_iterator end = vecnames.end();
                        for (; it != end; ++it)
-                               handle_package(p, trimSpaceAndEol(*it), options, 
+                               handle_package(p, trimSpaceAndEol(*it), options,
                                               in_lyx_preamble);
                }
 
                else if (t.cs() == "inputencoding") {
                        string const encoding = p.getArg('{','}');
                        h_inputencoding = encoding;
-                       p.setEncoding(encoding);
+                       p.setEncoding(encoding, Encoding::inputenc);
                }
 
                else if (t.cs() == "newenvironment") {
@@ -1252,10 +1556,30 @@ void Preamble::parse(Parser & p, string const & forceclass,
 
                }
 
+               else if (t.cs() == "newtheorem") {
+                       string const name = p.getArg('{', '}');
+                       string const opt1 = p.getFullOpt();
+                       string const opt2 = p.getFullOpt();
+                       string const body = p.verbatim_item();
+                       string const opt3 = p.getFullOpt();
+
+                       add_known_theorem(name, opt1, !opt2.empty(),
+                               from_utf8("\\newtheorem{" + name + '}' +
+                                         opt1 + opt2 + '{' + body + '}' + opt3));
+
+                       if (!in_lyx_preamble)
+                               h_preamble << "\\newtheorem{" << name << '}'
+                                          << opt1 << opt2 << '{' << '}' << opt3;
+               }
+
                else if (t.cs() == "def") {
                        string name = p.get_token().cs();
+                       // In fact, name may be more than the name:
+                       // In the test case of bug 8116
+                       // name == "csname SF@gobble@opt \endcsname".
+                       // Therefore, we need to use asInput() instead of cs().
                        while (p.next_token().cat() != catBegin)
-                               name += p.get_token().cs();
+                               name += p.get_token().asInput();
                        if (!in_lyx_preamble)
                                h_preamble << "\\def\\" << name << '{'
                                           << p.verbatim_item() << "}";
@@ -1319,6 +1643,19 @@ void Preamble::parse(Parser & p, string const & forceclass,
                else if (t.cs() == "setstretch")
                        h_spacing = "other " + p.verbatim_item();
 
+               else if (t.cs() == "synctex") {
+                       // the scheme is \synctex=value
+                       // where value can only be "1" or "-1"
+                       h_output_sync = "1";
+                       // there can be any character behind the value (e.g. a linebreak or a '\'
+                       // therefore we extract it char by char
+                       p.get_token();
+                       string value = p.get_token().asInput();
+                       if (value == "-")
+                               value += p.get_token().asInput();
+                       h_output_sync_macro = "\\synctex=" + value;
+               }
+
                else if (t.cs() == "begin") {
                        string const name = p.getArg('{', '}');
                        if (name == "document")
@@ -1354,6 +1691,9 @@ void Preamble::parse(Parser & p, string const & forceclass,
                        }
                }
 
+               else if (t.cs() == "bibliographystyle")
+                       h_biblio_style = p.verbatim_item();
+
                else if (t.cs() == "jurabibsetup") {
                        // FIXME p.getArg('{', '}') is most probably wrong (it
                        //       does not handle nested braces).
@@ -1436,8 +1776,6 @@ void Preamble::parse(Parser & p, string const & forceclass,
        // Force textclass if the user wanted it
        if (!forceclass.empty())
                h_textclass = forceclass;
-       if (noweb_mode && !prefixIs(h_textclass, "literate-"))
-               h_textclass.insert(0, "literate-");
        tc.setName(h_textclass);
        if (!tc.load()) {
                cerr << "Error: Could not read layout file for textclass \"" << h_textclass << "\"." << endl;
@@ -1448,6 +1786,18 @@ void Preamble::parse(Parser & p, string const & forceclass,
                ss << tc.sides();
                h_papersides = ss.str();
        }
+
+       // If the CJK package is used we cannot set the document language from
+       // the babel options. Instead, we guess which language is used most
+       // and set this one.
+       default_language = h_language;
+       if (is_full_document &&
+           (auto_packages.find("CJK") != auto_packages.end() ||
+            auto_packages.find("CJKutf8") != auto_packages.end())) {
+               p.pushPosition();
+               h_language = guessLanguage(p, default_language);
+               p.popPosition();
+       }
 }
 
 
@@ -1460,6 +1810,15 @@ string babel2lyx(string const & language)
 }
 
 
+string Preamble::polyglossia2lyx(string const & language)
+{
+       char const * const * where = is_known(language, polyglossia_languages);
+       if (where)
+               return coded_polyglossia_languages[where - polyglossia_languages];
+       return language;
+}
+
+
 string rgbcolor2code(string const & name)
 {
        char const * const * where = is_known(name, known_basic_colors);