src/tex2lyx/Preamble.cpp

   1 /**
   2  * \file Preamble.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Uwe Stöhr
   8  *
   9  * Full author contact details are available in file CREDITS.
  10  */
  11
  12 // {[(
  13
  14 #include <config.h>
  15
  16 #include "Preamble.h"
  17 #include "tex2lyx.h"
  18
  19 #include "Encoding.h"
  20 #include "LayoutFile.h"
  21 #include "Layout.h"
  22 #include "Lexer.h"
  23 #include "TextClass.h"
  24 #include "version.h"
  25
  26 #include "support/convert.h"
  27 #include "support/FileName.h"
  28 #include "support/filetools.h"
  29 #include "support/lstrings.h"
  30
  31 #include "support/regex.h"
  32
  33 #include <algorithm>
  34 #include <iostream>
  35
  36 using namespace std;
  37 using namespace lyx::support;
  38
  39
  40 namespace lyx {
  41
  42 Preamble preamble;
  43
  44 namespace {
  45
  46 // CJK languages are handled in text.cpp, polyglossia languages are listed
  47 // further down.
  48 /**
  49  * known babel language names (including synonyms)
  50  * not in standard babel: arabic, arabtex, armenian, belarusian, serbian-latin, thai
  51  * please keep this in sync with known_coded_languages line by line!
  52  */
  53 const char * const known_languages[] = {"acadian", "afrikaans", "albanian",
  54 "american", "arabic", "arabtex", "australian", "austrian", "bahasa", "bahasai",
  55 "bahasam", "basque", "belarusian", "bosnian", "brazil", "brazilian", "breton", "british",
  56 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
  57 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "francais",
  58 "french", "frenchb", "frenchle", "frenchpro", "friulan", "galician", "german", "germanb",
  59 "georgian", "greek", "hebrew", "hungarian", "icelandic", "indon", "indonesian",
  60 "interlingua", "irish", "italian", "japanese", "kazakh", "kurmanji", "latin",
  61 "latvian", "lithuanian", "lowersorbian", "lsorbian", "macedonian", "magyar", "malay", "meyalu",
  62 "mongolian", "naustrian", "newzealand", "ngerman", "ngermanb", "norsk", "nswissgerman",
  63 "nynorsk", "piedmontese", "polutonikogreek", "polish", "portuges", "portuguese",
  64 "romanian", "romansh", "russian", "russianb", "samin", "scottish", "serbian", "serbian-latin",
  65 "slovak", "slovene", "spanish", "swedish", "swissgerman", "thai", "turkish", "turkmen",
  66 "ukraineb", "ukrainian", "uppersorbian", "UKenglish", "USenglish", "usorbian",
  67 "vietnam", "welsh",
  68 0};
  69
  70 /**
  71  * the same as known_languages with .lyx names
  72  * please keep this in sync with known_languages line by line!
  73  */
  74 const char * const known_coded_languages[] = {"french", "afrikaans", "albanian",
  75 "american", "arabic_arabi", "arabic_arabtex", "australian", "austrian", "bahasa", "bahasa",
  76 "bahasam", "basque", "belarusian", "bosnian", "brazilian", "brazilian", "breton", "british",
  77 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
  78 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "french",
  79 "french", "french", "french", "french", "friulan", "galician", "german", "german",
  80 "georgian", "greek", "hebrew", "magyar", "icelandic", "bahasa", "bahasa",
  81 "interlingua", "irish", "italian", "japanese", "kazakh", "kurmanji", "latin",
  82 "latvian", "lithuanian", "lowersorbian", "lowersorbian", "macedonian", "magyar", "bahasam", "bahasam",
  83 "mongolian", "naustrian", "newzealand", "ngerman", "ngerman", "norsk", "german-ch",
  84 "nynorsk", "piedmontese", "polutonikogreek", "polish", "portuguese", "portuguese",
  85 "romanian", "romansh", "russian", "russian", "samin", "scottish", "serbian", "serbian-latin",
  86 "slovak", "slovene", "spanish", "swedish", "german-ch-old", "thai", "turkish", "turkmen",
  87 "ukrainian", "ukrainian", "uppersorbian", "english", "english", "uppersorbian",
  88 "vietnamese", "welsh",
  89 0};
  90
  91 /// languages with british quotes (.lyx names)
  92 const char * const known_british_quotes_languages[] = {"british", "welsh", 0};
  93
  94 /// languages with cjk quotes (.lyx names)
  95 const char * const known_cjk_quotes_languages[] = {"chinese-traditional",
  96 "japanese", "japanese-cjk", 0};
  97
  98 /// languages with cjk-angle quotes (.lyx names)
  99 const char * const known_cjkangle_quotes_languages[] = {"korean", 0};
 100
 101 /// languages with danish quotes (.lyx names)
 102 const char * const known_danish_quotes_languages[] = {"danish", 0};
 103
 104 /// languages with english quotes (.lyx names)
 105 const char * const known_english_quotes_languages[] = {"american", "australian",
 106 "bahasa", "bahasam", "brazilian", "canadian", "chinese-simplified", "english",
 107 "esperanto", "farsi", "interlingua", "irish", "newzealand", "scottish",
 108 "thai", "turkish", "vietnamese", 0};
 109
 110 /// languages with french quotes (.lyx names)
 111 const char * const known_french_quotes_languages[] = {"ancientgreek",
 112 "arabic_arabi", "arabic_arabtex", "asturian", "belarusian", "breton",
 113 "canadien", "catalan", "french", "friulan", "galician", "italian", "occitan",
 114 "piedmontese", "portuguese", "spanish", "spanish-mexico", 0};
 115
 116 /// languages with german quotes (.lyx names)
 117 const char * const known_german_quotes_languages[] = {"austrian", "bulgarian",
 118 "czech", "estonian", "georgian", "german", "icelandic", "latvian", "lithuanian",
 119 "lowersorbian", "macedonian", "naustrian", "ngerman", "romansh", "slovak", "slovene",
 120 "uppersorbian", 0};
 121
 122 /// languages with polish quotes (.lyx names)
 123 const char * const known_polish_quotes_languages[] = {"afrikaans", "bosnian", "croatian",
 124 "dutch", "magyar", "polish", "romanian", "serbian", "serbian-latin", 0};
 125
 126 /// languages with russian quotes (.lyx names)
 127 const char * const known_russian_quotes_languages[] = {"russian", "ukrainian", 0};
 128
 129 /// languages with swedish quotes (.lyx names)
 130 const char * const known_swedish_quotes_languages[] = {"finnish", "swedish", 0};
 131
 132 /// languages with swiss quotes (.lyx names)
 133 const char * const known_swiss_quotes_languages[] = {"albanian",
 134 "armenian", "basque", "german-ch", "german-ch-old",
 135 "norsk", "nynorsk", "turkmen", "ukrainian", "vietnamese", 0};
 136
 137 /// known language packages from the times before babel
 138 const char * const known_old_language_packages[] = {"french", "frenchle",
 139 "frenchpro", "german", "ngerman", "pmfrench", 0};
 140
 141 char const * const known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
 142
 143 const char * const known_roman_fonts[] = { "ae", "beraserif", "bookman",
 144 "ccfonts", "chancery", "charter", "cmr", "cochineal", "crimson", "fourier",
 145 "garamondx", "libertine", "libertineRoman", "libertine-type1", "lmodern", "mathdesign",
 146 "mathpazo", "mathptmx", "MinionPro", "newcent", "NotoSerif-TLF", "tgbonum", "tgchorus",
 147 "tgpagella", "tgschola", "tgtermes", "utopia", 0 };
 148
 149 const char * const known_sans_fonts[] = { "avant", "berasans", "biolinum",
 150 "biolinum-type1", "cmbr", "cmss", "helvet", "iwona", "iwonac", "iwonal", "iwonalc",
 151 "kurier", "kurierc", "kurierl", "kurierlc", "lmss", "NotoSans-TLF",
 152 "tgadventor", "tgheros", "uop", 0 };
 153
 154 const char * const known_typewriter_fonts[] = { "beramono", "cmtl", "cmtt",
 155 "courier", "lmtt", "luximono", "fourier", "libertineMono", "libertineMono-type1", "lmodern",
 156 "mathpazo", "mathptmx", "newcent", "NotoMono-TLF", "tgcursor", "txtt", 0 };
 157
 158 const char * const known_math_fonts[] = { "eulervm", "newtxmath", 0};
 159
 160 const char * const known_paper_sizes[] = { "a0paper", "b0paper", "c0paper",
 161 "a1paper", "b1paper", "c1paper", "a2paper", "b2paper", "c2paper", "a3paper",
 162 "b3paper", "c3paper", "a4paper", "b4paper", "c4paper", "a5paper", "b5paper",
 163 "c5paper", "a6paper", "b6paper", "c6paper", "executivepaper", "legalpaper",
 164 "letterpaper", "b0j", "b1j", "b2j", "b3j", "b4j", "b5j", "b6j", 0};
 165
 166 const char * const known_class_paper_sizes[] = { "a4paper", "a5paper",
 167 "executivepaper", "legalpaper", "letterpaper", 0};
 168
 169 const char * const known_paper_margins[] = { "lmargin", "tmargin", "rmargin",
 170 "bmargin", "headheight", "headsep", "footskip", "columnsep", 0};
 171
 172 const char * const known_coded_paper_margins[] = { "leftmargin", "topmargin",
 173 "rightmargin", "bottommargin", "headheight", "headsep", "footskip",
 174 "columnsep", 0};
 175
 176 /// commands that can start an \if...\else...\endif sequence
 177 const char * const known_if_commands[] = {"if", "ifarydshln", "ifbraket",
 178 "ifcancel", "ifcolortbl", "ifeurosym", "ifmarginnote", "ifmmode", "ifpdf",
 179 "ifsidecap", "ifupgreek", 0};
 180
 181 const char * const known_basic_colors[] = {"black", "blue", "brown", "cyan",
 182         "darkgray", "gray", "green", "lightgray", "lime", "magenta", "orange", "olive",
 183         "pink", "purple", "red", "teal", "violet", "white", "yellow", 0};
 184
 185 const char * const known_basic_color_codes[] = {"#000000", "#0000ff", "#964B00", "#00ffff",
 186         "#a9a9a9", "#808080", "#00ff00", "#d3d3d3", "#bfff00", "#ff00ff", "#ff7f00", "#808000",
 187         "#ffc0cb", "#800080", "#ff0000", "#008080", "#8f00ff", "#ffffff", "#ffff00", 0};
 188
 189 /// conditional commands with three arguments like \@ifundefined{}{}{}
 190 const char * const known_if_3arg_commands[] = {"@ifundefined", "IfFileExists",
 191 0};
 192
 193 /*!
 194  * Known file extensions for TeX files as used by \\includeonly
 195  */
 196 char const * const known_tex_extensions[] = {"tex", 0};
 197
 198 /// packages that work only in xetex
 199 /// polyglossia is handled separately
 200 const char * const known_xetex_packages[] = {"arabxetex", "fixlatvian",
 201 "fontbook", "fontwrap", "mathspec", "philokalia", "unisugar",
 202 "xeCJK", "xecolor", "xecyr", "xeindex", "xepersian", "xunicode", 0};
 203
 204 /// packages that are automatically skipped if loaded by LyX
 205 const char * const known_lyx_packages[] = {"amsbsy", "amsmath", "amssymb",
 206 "amstext", "amsthm", "array", "babel", "booktabs", "calc", "CJK", "color",
 207 "float", "fontspec", "framed", "graphicx", "hhline", "ifthen", "longtable",
 208 "makeidx", "minted", "multirow", "nomencl", "pdfpages", "prettyref", "refstyle",
 209 "rotating", "rotfloat", "splitidx", "setspace", "subscript", "textcomp", "tipa",
 210 "tipx", "tone", "ulem", "url", "varioref", "verbatim", "wrapfig", "xcolor",
 211 "xunicode", 0};
 212
 213 // codes used to remove packages that are loaded automatically by LyX.
 214 // Syntax: package_beg_sep<name>package_mid_sep<package loading code>package_end_sep
 215 const char package_beg_sep = '\001';
 216 const char package_mid_sep = '\002';
 217 const char package_end_sep = '\003';
 218
 219
 220 // returns true if at least one of the options in what has been found
 221 bool handle_opt(vector<string> & opts, char const * const * what, string & target)
 222 {
 223         if (opts.empty())
 224                 return false;
 225
 226         bool found = false;
 227         // the last language option is the document language (for babel and LyX)
 228         // the last size option is the document font size
 229         vector<string>::iterator it;
 230         vector<string>::iterator position = opts.begin();
 231         for (; *what; ++what) {
 232                 it = find(opts.begin(), opts.end(), *what);
 233                 if (it != opts.end()) {
 234                         if (it >= position) {
 235                                 found = true;
 236                                 target = *what;
 237                                 position = it;
 238                         }
 239                 }
 240         }
 241         return found;
 242 }
 243
 244
 245 void delete_opt(vector<string> & opts, char const * const * what)
 246 {
 247         if (opts.empty())
 248                 return;
 249
 250         // remove found options from the list
 251         // do this after handle_opt to avoid potential memory leaks
 252         vector<string>::iterator it;
 253         for (; *what; ++what) {
 254                 it = find(opts.begin(), opts.end(), *what);
 255                 if (it != opts.end())
 256                         opts.erase(it);
 257         }
 258 }
 259
 260
 261 /*!
 262  * Split a package options string (keyval format) into a vector.
 263  * Example input:
 264  *   authorformat=smallcaps,
 265  *   commabeforerest,
 266  *   titleformat=colonsep,
 267  *   bibformat={tabular,ibidem,numbered}
 268  */
 269 vector<string> split_options(string const & input)
 270 {
 271         vector<string> options;
 272         string option;
 273         Parser p(input);
 274         while (p.good()) {
 275                 Token const & t = p.get_token();
 276                 if (t.asInput() == ",") {
 277                         options.push_back(trimSpaceAndEol(option));
 278                         option.erase();
 279                 } else if (t.asInput() == "=") {
 280                         option += '=';
 281                         p.skip_spaces(true);
 282                         if (p.next_token().asInput() == "{")
 283                                 option += '{' + p.getArg('{', '}') + '}';
 284                 } else if (t.cat() != catSpace && t.cat() != catComment)
 285                         option += t.asInput();
 286         }
 287
 288         if (!option.empty())
 289                 options.push_back(trimSpaceAndEol(option));
 290
 291         return options;
 292 }
 293
 294
 295 /*!
 296  * Retrieve a keyval option "name={value with=sign}" named \p name from
 297  * \p options and return the value.
 298  * The found option is also removed from \p options.
 299  */
 300 string process_keyval_opt(vector<string> & options, string name)
 301 {
 302         for (size_t i = 0; i < options.size(); ++i) {
 303                 vector<string> option;
 304                 split(options[i], option, '=');
 305                 if (option.size() < 2)
 306                         continue;
 307                 if (option[0] == name) {
 308                         options.erase(options.begin() + i);
 309                         option.erase(option.begin());
 310                         return join(option, "=");
 311                 }
 312         }
 313         return "";
 314 }
 315
 316 } // anonymous namespace
 317
 318
 319 /**
 320  * known polyglossia language names (including variants)
 321  * FIXME: support spelling=old for german variants (german vs. ngerman LyX names etc)
 322  */
 323 const char * const Preamble::polyglossia_languages[] = {
 324 "albanian", "american", "amharic", "ancient", "arabic", "armenian", "asturian", "australian",
 325 "bahasai", "bahasam", "basque", "bengali", "brazil", "brazilian", "breton", "british", "bulgarian",
 326 "catalan", "coptic", "croatian", "czech", "danish", "divehi", "dutch",
 327 "english", "esperanto", "estonian", "farsi", "finnish", "french", "friulan",
 328 "galician", "greek", "monotonic", "hebrew", "hindi",
 329 "icelandic", "interlingua", "irish", "italian", "kannada", "khmer",
 330 "lao", "latin", "latvian", "lithuanian", "lsorbian", "magyar", "malayalam", "marathi",
 331 "austrian", "newzealand", "german", "norsk", "nynorsk", "occitan",
 332 "piedmontese", "polish", "polytonic", "portuges", "romanian", "romansh", "russian",
 333 "samin", "sanskrit", "scottish", "serbian", "slovak", "slovenian", "spanish", "swedish", "syriac",
 334 "tamil", "telugu", "thai", "tibetan", "turkish", "turkmen",
 335 "ukrainian", "urdu", "usorbian", "vietnamese", "welsh", 0};
 336 // not yet supported by LyX: "korean", "nko"
 337
 338 /**
 339  * the same as polyglossia_languages with .lyx names
 340  * please keep this in sync with polyglossia_languages line by line!
 341  */
 342 const char * const Preamble::coded_polyglossia_languages[] = {
 343 "albanian", "american", "amharic", "ancientgreek", "arabic_arabi", "armenian", "asturian", "australian",
 344 "bahasa", "bahasam", "basque", "bengali", "brazilian", "brazilian", "breton", "british", "bulgarian",
 345 "catalan", "coptic", "croatian", "czech", "danish", "divehi", "dutch",
 346 "english", "esperanto", "estonian", "farsi", "finnish", "french", "friulan",
 347 "galician", "greek", "greek", "hebrew", "hindi",
 348 "icelandic", "interlingua", "irish", "italian", "kannada", "khmer",
 349 "lao", "latin", "latvian", "lithuanian", "lowersorbian", "magyar", "malayalam", "marathi",
 350 "naustrian","newzealand", "ngerman", "norsk", "nynorsk", "occitan",
 351 "piedmontese", "polish", "polutonikogreek", "portuges", "romanian", "romansh", "russian",
 352 "samin", "sanskrit", "scottish", "serbian", "slovak", "slovene", "spanish", "swedish", "syriac",
 353 "tamil", "telugu", "thai", "tibetan", "turkish", "turkmen",
 354 "ukrainian", "urdu", "uppersorbian", "vietnamese", "welsh", 0};
 355 // not yet supported by LyX: "korean-polyglossia", "nko"
 356
 357
 358 bool Preamble::usePolyglossia() const
 359 {
 360         return h_use_non_tex_fonts && h_language_package == "default";
 361 }
 362
 363
 364 bool Preamble::indentParagraphs() const
 365 {
 366         return h_paragraph_separation == "indent";
 367 }
 368
 369
 370 bool Preamble::isPackageUsed(string const & package) const
 371 {
 372         return used_packages.find(package) != used_packages.end();
 373 }
 374
 375
 376 vector<string> Preamble::getPackageOptions(string const & package) const
 377 {
 378         map<string, vector<string> >::const_iterator it = used_packages.find(package);
 379         if (it != used_packages.end())
 380                 return it->second;
 381         return vector<string>();
 382 }
 383
 384
 385 void Preamble::registerAutomaticallyLoadedPackage(std::string const & package)
 386 {
 387         auto_packages.insert(package);
 388 }
 389
 390
 391 void Preamble::addModule(string const & module)
 392 {
 393         used_modules.push_back(module);
 394 }
 395
 396
 397 void Preamble::suppressDate(bool suppress)
 398 {
 399         if (suppress)
 400                 h_suppress_date = "true";
 401         else
 402                 h_suppress_date = "false";
 403 }
 404
 405
 406 void Preamble::registerAuthor(std::string const & name)
 407 {
 408         Author author(from_utf8(name), empty_docstring());
 409         author.setUsed(true);
 410         authors_.record(author);
 411         h_tracking_changes = "true";
 412         h_output_changes = "true";
 413 }
 414
 415
 416 Author const & Preamble::getAuthor(std::string const & name) const
 417 {
 418         Author author(from_utf8(name), empty_docstring());
 419         for (AuthorList::Authors::const_iterator it = authors_.begin();
 420              it != authors_.end(); ++it)
 421                 if (*it == author)
 422                         return *it;
 423         static Author const dummy;
 424         return dummy;
 425 }
 426
 427
 428 int Preamble::getSpecialTableColumnArguments(char c) const
 429 {
 430         map<char, int>::const_iterator it = special_columns_.find(c);
 431         if (it == special_columns_.end())
 432                 return -1;
 433         return it->second;
 434 }
 435
 436
 437 void Preamble::add_package(string const & name, vector<string> & options)
 438 {
 439         // every package inherits the global options
 440         if (used_packages.find(name) == used_packages.end())
 441                 used_packages[name] = split_options(h_options);
 442
 443         // Insert options passed via PassOptionsToPackage
 444         for (auto const & p : extra_package_options_) {
 445                 if (p.first == name) {
 446                         vector<string> eo = getVectorFromString(p.second);
 447                         for (auto const & eoi : eo)
 448                                 options.push_back(eoi);
 449                 }
 450
 451         }
 452         vector<string> & v = used_packages[name];
 453         v.insert(v.end(), options.begin(), options.end());
 454         if (name == "jurabib") {
 455                 // Don't output the order argument (see the cite command
 456                 // handling code in text.cpp).
 457                 vector<string>::iterator end =
 458                         remove(options.begin(), options.end(), "natbiborder");
 459                 end = remove(options.begin(), end, "jurabiborder");
 460                 options.erase(end, options.end());
 461         }
 462 }
 463
 464
 465 namespace {
 466
 467 // Given is a string like "scaled=0.9" or "Scale=0.9", return 0.9 * 100
 468 bool scale_as_percentage(string const & scale, string & percentage)
 469 {
 470         string::size_type pos = scale.find('=');
 471         if (pos != string::npos) {
 472                 string value = scale.substr(pos + 1);
 473                 if (isStrDbl(value)) {
 474                         percentage = convert<string>(
 475                                 static_cast<int>(100 * convert<double>(value)));
 476                         return true;
 477                 }
 478         }
 479         return false;
 480 }
 481
 482
 483 string remove_braces(string const & value)
 484 {
 485         if (value.empty())
 486                 return value;
 487         if (value[0] == '{' && value[value.length()-1] == '}')
 488                 return value.substr(1, value.length()-2);
 489         return value;
 490 }
 491
 492 } // anonymous namespace
 493
 494
 495 Preamble::Preamble() : one_language(true), explicit_babel(false),
 496         title_layout_found(false), index_number(0), h_font_cjk_set(false),
 497         h_use_microtype("false")
 498 {
 499         //h_backgroundcolor;
 500         //h_boxbgcolor;
 501         h_biblio_style            = "plain";
 502         h_bibtex_command          = "default";
 503         h_cite_engine             = "basic";
 504         h_cite_engine_type        = "default";
 505         h_color                   = "#008000";
 506         h_defskip                 = "medskip";
 507         h_dynamic_quotes          = false;
 508         //h_float_placement;
 509         //h_fontcolor;
 510         h_fontencoding            = "default";
 511         h_font_roman[0]           = "default";
 512         h_font_roman[1]           = "default";
 513         h_font_sans[0]            = "default";
 514         h_font_sans[1]            = "default";
 515         h_font_typewriter[0]      = "default";
 516         h_font_typewriter[1]      = "default";
 517         h_font_math[0]            = "auto";
 518         h_font_math[1]            = "auto";
 519         h_font_default_family     = "default";
 520         h_use_non_tex_fonts       = false;
 521         h_font_sc                 = "false";
 522         h_font_osf                = "false";
 523         h_font_sf_scale[0]        = "100";
 524         h_font_sf_scale[1]        = "100";
 525         h_font_tt_scale[0]        = "100";
 526         h_font_tt_scale[1]        = "100";
 527         //h_font_cjk
 528         h_is_mathindent           = "0";
 529         h_math_numbering_side     = "default";
 530         h_graphics                = "default";
 531         h_default_output_format   = "default";
 532         h_html_be_strict          = "false";
 533         h_html_css_as_file        = "0";
 534         h_html_math_output        = "0";
 535         h_index[0]                = "Index";
 536         h_index_command           = "default";
 537         h_inputencoding           = "auto";
 538         h_justification           = "true";
 539         h_language                = "english";
 540         h_language_package        = "none";
 541         //h_listings_params;
 542         h_maintain_unincluded_children = "false";
 543         //h_margins;
 544         //h_notefontcolor;
 545         //h_options;
 546         h_output_changes          = "false";
 547         h_output_sync             = "0";
 548         //h_output_sync_macro
 549         h_papercolumns            = "1";
 550         h_paperfontsize           = "default";
 551         h_paperorientation        = "portrait";
 552         h_paperpagestyle          = "default";
 553         //h_papersides;
 554         h_papersize               = "default";
 555         h_paragraph_indentation   = "default";
 556         h_paragraph_separation    = "indent";
 557         //h_pdf_title;
 558         //h_pdf_author;
 559         //h_pdf_subject;
 560         //h_pdf_keywords;
 561         h_pdf_bookmarks           = "0";
 562         h_pdf_bookmarksnumbered   = "0";
 563         h_pdf_bookmarksopen       = "0";
 564         h_pdf_bookmarksopenlevel  = "1";
 565         h_pdf_breaklinks          = "0";
 566         h_pdf_pdfborder           = "0";
 567         h_pdf_colorlinks          = "0";
 568         h_pdf_backref             = "section";
 569         h_pdf_pdfusetitle         = "0";
 570         //h_pdf_pagemode;
 571         //h_pdf_quoted_options;
 572         h_quotes_style         = "english";
 573         h_secnumdepth             = "3";
 574         h_shortcut[0]             = "idx";
 575         h_spacing                 = "single";
 576         h_save_transient_properties = "true";
 577         h_suppress_date           = "false";
 578         h_textclass               = "article";
 579         h_tocdepth                = "3";
 580         h_tracking_changes        = "false";
 581         h_use_bibtopic            = "false";
 582         h_use_dash_ligatures      = "true";
 583         h_use_indices             = "false";
 584         h_use_geometry            = "false";
 585         h_use_default_options     = "false";
 586         h_use_hyperref            = "false";
 587         h_use_microtype           = "false";
 588         h_use_refstyle            = false;
 589         h_use_minted              = false;
 590         h_use_packages["amsmath"]    = "1";
 591         h_use_packages["amssymb"]    = "0";
 592         h_use_packages["cancel"]     = "0";
 593         h_use_packages["esint"]      = "1";
 594         h_use_packages["mhchem"]     = "0";
 595         h_use_packages["mathdots"]   = "0";
 596         h_use_packages["mathtools"]  = "0";
 597         h_use_packages["stackrel"]   = "0";
 598         h_use_packages["stmaryrd"]   = "0";
 599         h_use_packages["undertilde"] = "0";
 600 }
 601
 602
 603 void Preamble::handle_hyperref(vector<string> & options)
 604 {
 605         // FIXME swallow inputencoding changes that might surround the
 606         //       hyperref setup if it was written by LyX
 607         h_use_hyperref = "true";
 608         // swallow "unicode=true", since LyX does always write that
 609         vector<string>::iterator it =
 610                 find(options.begin(), options.end(), "unicode=true");
 611         if (it != options.end())
 612                 options.erase(it);
 613         it = find(options.begin(), options.end(), "pdfusetitle");
 614         if (it != options.end()) {
 615                 h_pdf_pdfusetitle = "1";
 616                 options.erase(it);
 617         }
 618         string bookmarks = process_keyval_opt(options, "bookmarks");
 619         if (bookmarks == "true")
 620                 h_pdf_bookmarks = "1";
 621         else if (bookmarks == "false")
 622                 h_pdf_bookmarks = "0";
 623         if (h_pdf_bookmarks == "1") {
 624                 string bookmarksnumbered =
 625                         process_keyval_opt(options, "bookmarksnumbered");
 626                 if (bookmarksnumbered == "true")
 627                         h_pdf_bookmarksnumbered = "1";
 628                 else if (bookmarksnumbered == "false")
 629                         h_pdf_bookmarksnumbered = "0";
 630                 string bookmarksopen =
 631                         process_keyval_opt(options, "bookmarksopen");
 632                 if (bookmarksopen == "true")
 633                         h_pdf_bookmarksopen = "1";
 634                 else if (bookmarksopen == "false")
 635                         h_pdf_bookmarksopen = "0";
 636                 if (h_pdf_bookmarksopen == "1") {
 637                         string bookmarksopenlevel =
 638                                 process_keyval_opt(options, "bookmarksopenlevel");
 639                         if (!bookmarksopenlevel.empty())
 640                                 h_pdf_bookmarksopenlevel = bookmarksopenlevel;
 641                 }
 642         }
 643         string breaklinks = process_keyval_opt(options, "breaklinks");
 644         if (breaklinks == "true")
 645                 h_pdf_breaklinks = "1";
 646         else if (breaklinks == "false")
 647                 h_pdf_breaklinks = "0";
 648         string pdfborder = process_keyval_opt(options, "pdfborder");
 649         if (pdfborder == "{0 0 0}")
 650                 h_pdf_pdfborder = "1";
 651         else if (pdfborder == "{0 0 1}")
 652                 h_pdf_pdfborder = "0";
 653         string backref = process_keyval_opt(options, "backref");
 654         if (!backref.empty())
 655                 h_pdf_backref = backref;
 656         string colorlinks = process_keyval_opt(options, "colorlinks");
 657         if (colorlinks == "true")
 658                 h_pdf_colorlinks = "1";
 659         else if (colorlinks == "false")
 660                 h_pdf_colorlinks = "0";
 661         string pdfpagemode = process_keyval_opt(options, "pdfpagemode");
 662         if (!pdfpagemode.empty())
 663                 h_pdf_pagemode = pdfpagemode;
 664         string pdftitle = process_keyval_opt(options, "pdftitle");
 665         if (!pdftitle.empty()) {
 666                 h_pdf_title = remove_braces(pdftitle);
 667         }
 668         string pdfauthor = process_keyval_opt(options, "pdfauthor");
 669         if (!pdfauthor.empty()) {
 670                 h_pdf_author = remove_braces(pdfauthor);
 671         }
 672         string pdfsubject = process_keyval_opt(options, "pdfsubject");
 673         if (!pdfsubject.empty())
 674                 h_pdf_subject = remove_braces(pdfsubject);
 675         string pdfkeywords = process_keyval_opt(options, "pdfkeywords");
 676         if (!pdfkeywords.empty())
 677                 h_pdf_keywords = remove_braces(pdfkeywords);
 678         if (!options.empty()) {
 679                 if (!h_pdf_quoted_options.empty())
 680                         h_pdf_quoted_options += ',';
 681                 h_pdf_quoted_options += join(options, ",");
 682                 options.clear();
 683         }
 684 }
 685
 686
 687 void Preamble::handle_geometry(vector<string> & options)
 688 {
 689         h_use_geometry = "true";
 690         vector<string>::iterator it;
 691         // paper orientation
 692         if ((it = find(options.begin(), options.end(), "landscape")) != options.end()) {
 693                 h_paperorientation = "landscape";
 694                 options.erase(it);
 695         }
 696         // paper size
 697         // keyval version: "paper=letter"
 698         string paper = process_keyval_opt(options, "paper");
 699         if (!paper.empty())
 700                 h_papersize = paper + "paper";
 701         // alternative version: "letterpaper"
 702         handle_opt(options, known_paper_sizes, h_papersize);
 703         delete_opt(options, known_paper_sizes);
 704         // page margins
 705         char const * const * margin = known_paper_margins;
 706         for (; *margin; ++margin) {
 707                 string value = process_keyval_opt(options, *margin);
 708                 if (!value.empty()) {
 709                         int k = margin - known_paper_margins;
 710                         string name = known_coded_paper_margins[k];
 711                         h_margins += '\\' + name + ' ' + value + '\n';
 712                 }
 713         }
 714 }
 715
 716
 717 void Preamble::handle_package(Parser &p, string const & name,
 718                               string const & opts, bool in_lyx_preamble,
 719                               bool detectEncoding)
 720 {
 721         vector<string> options = split_options(opts);
 722         add_package(name, options);
 723
 724         if (is_known(name, known_xetex_packages)) {
 725                 xetex = true;
 726                 h_use_non_tex_fonts = true;
 727                 registerAutomaticallyLoadedPackage("fontspec");
 728                 if (h_inputencoding == "auto")
 729                         p.setEncoding("UTF-8");
 730         }
 731
 732         // roman fonts
 733         if (is_known(name, known_roman_fonts))
 734                 h_font_roman[0] = name;
 735
 736         if (name == "fourier") {
 737                 h_font_roman[0] = "utopia";
 738                 // when font uses real small capitals
 739                 if (opts == "expert")
 740                         h_font_sc = "true";
 741         }
 742
 743         if (name == "garamondx") {
 744                 h_font_roman[0] = "garamondx";
 745                 if (opts == "osfI")
 746                         h_font_osf = "true";
 747         }
 748
 749         if (name == "libertine") {
 750                 h_font_roman[0] = "libertine";
 751                 // this automatically invokes biolinum
 752                 h_font_sans[0] = "biolinum";
 753                 // as well as libertineMono
 754                 h_font_typewriter[0] = "libertine-mono";
 755                 if (opts == "osf")
 756                         h_font_osf = "true";
 757                 else if (opts == "lining")
 758                         h_font_osf = "false";
 759         }
 760
 761         if (name == "libertineRoman" || name == "libertine-type1") {
 762                 h_font_roman[0] = "libertine";
 763                 // NOTE: contrary to libertine.sty, libertineRoman
 764                 // and libertine-type1 do not automatically invoke
 765                 // biolinum and libertineMono
 766                 if (opts == "lining")
 767                         h_font_osf = "false";
 768                 else if (opts == "osf")
 769                         h_font_osf = "true";
 770         }
 771
 772         if (name == "MinionPro") {
 773                 h_font_roman[0] = "minionpro";
 774                 if (opts.find("lf") != string::npos)
 775                         h_font_osf = "false";
 776                 else
 777                         h_font_osf = "true";
 778                 if (opts.find("onlytext") != string::npos)
 779                         h_font_math[0] = "default";
 780                 else
 781                         h_font_math[0] = "auto";
 782         }
 783
 784         if (name == "mathdesign") {
 785                 if (opts.find("charter") != string::npos)
 786                         h_font_roman[0] = "md-charter";
 787                 if (opts.find("garamond") != string::npos)
 788                         h_font_roman[0] = "md-garamond";
 789                 if (opts.find("utopia") != string::npos)
 790                         h_font_roman[0] = "md-utopia";
 791                 if (opts.find("expert") != string::npos) {
 792                         h_font_sc = "true";
 793                         h_font_osf = "true";
 794                 }
 795         }
 796
 797         else if (name == "mathpazo")
 798                 h_font_roman[0] = "palatino";
 799
 800         else if (name == "mathptmx")
 801                 h_font_roman[0] = "times";
 802
 803         if (name == "crimson")
 804                 h_font_roman[0] = "cochineal";
 805
 806         if (name == "cochineal") {
 807                 h_font_roman[0] = "cochineal";
 808                 // cochineal can have several options, e.g. [proportional,osf]
 809                 string::size_type pos = opts.find("osf");
 810                 if (pos != string::npos)
 811                         h_font_osf = "true";
 812         }
 813
 814         if (name == "noto") {
 815                 // noto can have several options
 816                 if (opts.empty())
 817                         h_font_roman[0] = "NotoSerif-TLF";
 818                 string::size_type pos = opts.find("rm");
 819                 if (pos != string::npos)
 820                         h_font_roman[0] = "NotoSerif-TLF";
 821                 pos = opts.find("sf");
 822                 if (pos != string::npos)
 823                         h_font_sans[0] = "NotoSans-TLF";
 824                 pos = opts.find("nott");
 825                 if (pos != string::npos) {
 826                         h_font_roman[0] = "NotoSerif-TLF";
 827                         h_font_sans[0] = "NotoSans-TLF";
 828                 }
 829                 // noto as typewriter is handled in handling of \ttdefault
 830                 // special cases are handled in handling of \rmdefault and \sfdefault
 831         }
 832
 833         // sansserif fonts
 834         if (is_known(name, known_sans_fonts)) {
 835                 h_font_sans[0] = name;
 836                 if (options.size() >= 1) {
 837                         if (scale_as_percentage(opts, h_font_sf_scale[0]))
 838                                 options.clear();
 839                 }
 840         }
 841
 842         if (name == "biolinum" || name == "biolinum-type1") {
 843                 h_font_sans[0] = "biolinum";
 844                 // biolinum can have several options, e.g. [osf,scaled=0.97]
 845                 string::size_type pos = opts.find("osf");
 846                 if (pos != string::npos)
 847                         h_font_osf = "true";
 848         }
 849
 850         // typewriter fonts
 851         if (is_known(name, known_typewriter_fonts)) {
 852                 // fourier can be set as roman font _only_
 853                 // fourier as typewriter is handled in handling of \ttdefault
 854                 if (name != "fourier") {
 855                         h_font_typewriter[0] = name;
 856                         if (options.size() >= 1) {
 857                                 if (scale_as_percentage(opts, h_font_tt_scale[0]))
 858                                         options.clear();
 859                         }
 860                 }
 861         }
 862
 863         if (name == "libertineMono" || name == "libertineMono-type1")
 864                 h_font_typewriter[0] = "libertine-mono";
 865
 866         // font uses old-style figure
 867         if (name == "eco")
 868                 h_font_osf = "true";
 869
 870         // math fonts
 871         if (is_known(name, known_math_fonts))
 872                 h_font_math[0] = name;
 873
 874         if (name == "newtxmath") {
 875                 if (opts.empty())
 876                         h_font_math[0] = "newtxmath";
 877                 else if (opts == "garamondx")
 878                         h_font_math[0] = "garamondx-ntxm";
 879                 else if (opts == "libertine")
 880                         h_font_math[0] = "libertine-ntxm";
 881                 else if (opts == "minion")
 882                         h_font_math[0] = "minion-ntxm";
 883                 else if (opts == "cochineal")
 884                         h_font_math[0] = "cochineal-ntxm";
 885         }
 886
 887         if (name == "iwona")
 888                 if (opts == "math")
 889                         h_font_math[0] = "iwona-math";
 890
 891         if (name == "kurier")
 892                 if (opts == "math")
 893                         h_font_math[0] = "kurier-math";
 894
 895         // after the detection and handling of special cases, we can remove the
 896         // fonts, otherwise they would appear in the preamble, see bug #7856
 897         if (is_known(name, known_roman_fonts) || is_known(name, known_sans_fonts)
 898                 ||      is_known(name, known_typewriter_fonts) || is_known(name, known_math_fonts))
 899                 ;
 900         //"On". See the enum Package in BufferParams.h if you thought that "2" should have been "42"
 901         else if (name == "amsmath" || name == "amssymb" || name == "cancel" ||
 902                  name == "esint" || name == "mhchem" || name == "mathdots" ||
 903                  name == "mathtools" || name == "stackrel" ||
 904                  name == "stmaryrd" || name == "undertilde")
 905                 h_use_packages[name] = "2";
 906
 907         else if (name == "babel") {
 908                 h_language_package = "default";
 909                 // One might think we would have to do nothing if babel is loaded
 910                 // without any options to prevent pollution of the preamble with this
 911                 // babel call in every roundtrip.
 912                 // But the user could have defined babel-specific things afterwards. So
 913                 // we need to keep it in the preamble to prevent cases like bug #7861.
 914                 if (!opts.empty()) {
 915                         // check if more than one option was used - used later for inputenc
 916                         if (options.begin() != options.end() - 1)
 917                                 one_language = false;
 918                         // babel takes the last language of the option of its \usepackage
 919                         // call as document language. If there is no such language option, the
 920                         // last language in the documentclass options is used.
 921                         handle_opt(options, known_languages, h_language);
 922                         // translate the babel name to a LyX name
 923                         h_language = babel2lyx(h_language);
 924                         if (h_language == "japanese") {
 925                                 // For Japanese, the encoding isn't indicated in the source
 926                                 // file, and there's really not much we can do. We could
 927                                 // 1) offer a list of possible encodings to choose from, or
 928                                 // 2) determine the encoding of the file by inspecting it.
 929                                 // For the time being, we leave the encoding alone so that
 930                                 // we don't get iconv errors when making a wrong guess, and
 931                                 // we will output a note at the top of the document
 932                                 // explaining what to do.
 933                                 Encoding const * const enc = encodings.fromIconvName(
 934                                         p.getEncoding(), Encoding::japanese, false);
 935                                 if (enc)
 936                                         h_inputencoding = enc->name();
 937                                 is_nonCJKJapanese = true;
 938                                 // in this case babel can be removed from the preamble
 939                                 registerAutomaticallyLoadedPackage("babel");
 940                         } else {
 941                                 // If babel is called with options, LyX puts them by default into the
 942                                 // document class options. This works for most languages, except
 943                                 // for Latvian, Lithuanian, Mongolian, Turkmen and Vietnamese and
 944                                 // perhaps in future others.
 945                                 // Therefore keep the babel call as it is as the user might have
 946                                 // reasons for it.
 947                                 string const babelcall = "\\usepackage[" + opts + "]{babel}\n";
 948                                 if (!contains(h_preamble.str(), babelcall))
 949                                         h_preamble << babelcall;
 950                         }
 951                         delete_opt(options, known_languages);
 952                 } else {
 953                         if (!contains(h_preamble.str(), "\\usepackage{babel}\n"))
 954                                 h_preamble << "\\usepackage{babel}\n";
 955                         explicit_babel = true;
 956                 }
 957         }
 958
 959         else if (name == "polyglossia") {
 960                 h_language_package = "default";
 961                 h_default_output_format = "pdf4";
 962                 h_use_non_tex_fonts = true;
 963                 xetex = true;
 964                 registerAutomaticallyLoadedPackage("xunicode");
 965                 if (h_inputencoding == "auto")
 966                         p.setEncoding("UTF-8");
 967         }
 968
 969         else if (name == "CJK") {
 970                 // set the encoding to "auto" because it might be set to "default" by the babel handling
 971                 // and this would not be correct for CJK
 972                 if (h_inputencoding == "default")
 973                         h_inputencoding = "auto";
 974                 registerAutomaticallyLoadedPackage("CJK");
 975         }
 976
 977         else if (name == "CJKutf8") {
 978                 h_inputencoding = "utf8-cjk";
 979                 p.setEncoding("UTF-8");
 980                 registerAutomaticallyLoadedPackage("CJKutf8");
 981         }
 982
 983         else if (name == "fontenc") {
 984                 h_fontencoding = getStringFromVector(options, ",");
 985                 /* We could do the following for better round trip support,
 986                  * but this makes the document less portable, so I skip it:
 987                 if (h_fontencoding == lyxrc.fontenc)
 988                         h_fontencoding = "global";
 989                  */
 990                 options.clear();
 991         }
 992
 993         else if (name == "inputenc" || name == "luainputenc") {
 994                 // h_inputencoding is only set when there is not more than one
 995                 // inputenc option because otherwise h_inputencoding must be
 996                 // set to "auto" (the default encoding of the document language)
 997                 // Therefore check that exactly one option is passed to inputenc.
 998                 // It is also only set when there is not more than one babel
 999                 // language option.
1000                 if (!options.empty()) {
1001                         string const encoding = options.back();
1002                         Encoding const * const enc = encodings.fromLaTeXName(
1003                                 encoding, Encoding::inputenc, true);
1004                         if (!enc) {
1005                                 if (!detectEncoding)
1006                                         cerr << "Unknown encoding " << encoding
1007                                              << ". Ignoring." << std::endl;
1008                         } else {
1009                                 if (!enc->unsafe() && options.size() == 1 && one_language == true)
1010                                         h_inputencoding = enc->name();
1011                                 p.setEncoding(enc->iconvName());
1012                         }
1013                         options.clear();
1014                 }
1015         }
1016
1017         else if (name == "srcltx") {
1018                 h_output_sync = "1";
1019                 if (!opts.empty()) {
1020                         h_output_sync_macro = "\\usepackage[" + opts + "]{srcltx}";
1021                         options.clear();
1022                 } else
1023                         h_output_sync_macro = "\\usepackage{srcltx}";
1024         }
1025
1026         else if (is_known(name, known_old_language_packages)) {
1027                 // known language packages from the times before babel
1028                 // if they are found and not also babel, they will be used as
1029                 // custom language package
1030                 h_language_package = "\\usepackage{" + name + "}";
1031         }
1032
1033         else if (name == "lyxskak") {
1034                 // ignore this and its options
1035                 const char * const o[] = {"ps", "mover", 0};
1036                 delete_opt(options, o);
1037         }
1038
1039         else if (is_known(name, known_lyx_packages) && options.empty()) {
1040                 if (name == "splitidx")
1041                         h_use_indices = "true";
1042                 else if (name == "minted")
1043                         h_use_minted = true;
1044                 else if (name == "refstyle")
1045                         h_use_refstyle = true;
1046                 else if (name == "prettyref")
1047                         h_use_refstyle = false;
1048                 if (!in_lyx_preamble) {
1049                         h_preamble << package_beg_sep << name
1050                                    << package_mid_sep << "\\usepackage{"
1051                                    << name << '}';
1052                         if (p.next_token().cat() == catNewline ||
1053                             (p.next_token().cat() == catSpace &&
1054                              p.next_next_token().cat() == catNewline))
1055                                 h_preamble << '\n';
1056                         h_preamble << package_end_sep;
1057                 }
1058         }
1059
1060         else if (name == "geometry")
1061                 handle_geometry(options);
1062
1063         else if (name == "subfig")
1064                 ; // ignore this FIXME: Use the package separator mechanism instead
1065
1066         else if (char const * const * where = is_known(name, known_languages))
1067                 h_language = known_coded_languages[where - known_languages];
1068
1069         else if (name == "natbib") {
1070                 h_biblio_style = "plainnat";
1071                 h_cite_engine = "natbib";
1072                 h_cite_engine_type = "authoryear";
1073                 vector<string>::iterator it =
1074                         find(options.begin(), options.end(), "authoryear");
1075                 if (it != options.end())
1076                         options.erase(it);
1077                 else {
1078                         it = find(options.begin(), options.end(), "numbers");
1079                         if (it != options.end()) {
1080                                 h_cite_engine_type = "numerical";
1081                                 options.erase(it);
1082                         }
1083                 }
1084                 if (!options.empty())
1085                         h_biblio_options = join(options, ",");
1086         }
1087
1088         else if (name == "biblatex") {
1089                 h_biblio_style = "plainnat";
1090                 h_cite_engine = "biblatex";
1091                 h_cite_engine_type = "authoryear";
1092                 string opt;
1093                 vector<string>::iterator it =
1094                         find(options.begin(), options.end(), "natbib");
1095                 if (it != options.end()) {
1096                         options.erase(it);
1097                         h_cite_engine = "biblatex-natbib";
1098                 } else {
1099                         opt = process_keyval_opt(options, "natbib");
1100                         if (opt == "true")
1101                                 h_cite_engine = "biblatex-natbib";
1102                 }
1103                 opt = process_keyval_opt(options, "style");
1104                 if (!opt.empty()) {
1105                         h_biblatex_citestyle = opt;
1106                         h_biblatex_bibstyle = opt;
1107                 } else {
1108                         opt = process_keyval_opt(options, "citestyle");
1109                         if (!opt.empty())
1110                                 h_biblatex_citestyle = opt;
1111                         opt = process_keyval_opt(options, "bibstyle");
1112                         if (!opt.empty())
1113                                 h_biblatex_bibstyle = opt;
1114                 }
1115                 opt = process_keyval_opt(options, "refsection");
1116                 if (!opt.empty()) {
1117                         if (opt == "none" || opt == "part"
1118                             || opt == "chapter" || opt == "section"
1119                             || opt == "subsection")
1120                                 h_multibib = opt;
1121                         else
1122                                 cerr << "Ignoring unkown refesection value '"
1123                                      << opt << "'.";
1124                 }
1125                 if (!options.empty()) {
1126                         h_biblio_options = join(options, ",");
1127                         options.clear();
1128                 }
1129         }
1130
1131         else if (name == "jurabib") {
1132                 h_biblio_style = "jurabib";
1133                 h_cite_engine = "jurabib";
1134                 h_cite_engine_type = "authoryear";
1135                 if (!options.empty())
1136                         h_biblio_options = join(options, ",");
1137         }
1138
1139         else if (name == "bibtopic")
1140                 h_use_bibtopic = "true";
1141
1142         else if (name == "chapterbib")
1143                 h_multibib = "child";
1144
1145         else if (name == "hyperref")
1146                 handle_hyperref(options);
1147
1148         else if (name == "algorithm2e") {
1149                 // Load "algorithm2e" module
1150                 addModule("algorithm2e");
1151                 // Add the package options to the global document options
1152                 if (!options.empty()) {
1153                         if (h_options.empty())
1154                                 h_options = join(options, ",");
1155                         else
1156                                 h_options += ',' + join(options, ",");
1157                 }
1158         }
1159         else if (name == "microtype") {
1160                 //we internally support only microtype without params
1161                 if (options.empty())
1162                         h_use_microtype = "true";
1163                 else
1164                         h_preamble << "\\usepackage[" << opts << "]{microtype}";
1165         }
1166
1167         else if (!in_lyx_preamble) {
1168                 if (options.empty())
1169                         h_preamble << "\\usepackage{" << name << '}';
1170                 else {
1171                         h_preamble << "\\usepackage[" << opts << "]{"
1172                                    << name << '}';
1173                         options.clear();
1174                 }
1175                 if (p.next_token().cat() == catNewline ||
1176                     (p.next_token().cat() == catSpace &&
1177                      p.next_next_token().cat() == catNewline))
1178                         h_preamble << '\n';
1179         }
1180
1181         // We need to do something with the options...
1182         if (!options.empty() && !detectEncoding)
1183                 cerr << "Ignoring options '" << join(options, ",")
1184                      << "' of package " << name << '.' << endl;
1185
1186         // remove the whitespace
1187         p.skip_spaces();
1188 }
1189
1190
1191 void Preamble::handle_if(Parser & p, bool in_lyx_preamble)
1192 {
1193         while (p.good()) {
1194                 Token t = p.get_token();
1195                 if (t.cat() == catEscape &&
1196                     is_known(t.cs(), known_if_commands))
1197                         handle_if(p, in_lyx_preamble);
1198                 else {
1199                         if (!in_lyx_preamble)
1200                                 h_preamble << t.asInput();
1201                         if (t.cat() == catEscape && t.cs() == "fi")
1202                                 return;
1203                 }
1204         }
1205 }
1206
1207
1208 bool Preamble::writeLyXHeader(ostream & os, bool subdoc, string const & outfiledir)
1209 {
1210         if (contains(h_float_placement, "H"))
1211                 registerAutomaticallyLoadedPackage("float");
1212         if (h_spacing != "single" && h_spacing != "default")
1213                 registerAutomaticallyLoadedPackage("setspace");
1214         if (h_use_packages["amsmath"] == "2") {
1215                 // amsbsy and amstext are already provided by amsmath
1216                 registerAutomaticallyLoadedPackage("amsbsy");
1217                 registerAutomaticallyLoadedPackage("amstext");
1218         }
1219
1220         // output the LyX file settings
1221         // Important: Keep the version formatting in sync with LyX and
1222         //            lyx2lyx (bug 7951)
1223         string const origin = roundtripMode() ? "roundtrip" : outfiledir;
1224         os << "#LyX file created by tex2lyx " << lyx_version_major << '.'
1225            << lyx_version_minor << '\n'
1226            << "\\lyxformat " << LYX_FORMAT << '\n'
1227            << "\\begin_document\n"
1228            << "\\begin_header\n"
1229            << "\\save_transient_properties " << h_save_transient_properties << "\n"
1230            << "\\origin " << origin << "\n"
1231            << "\\textclass " << h_textclass << "\n";
1232         string const raw = subdoc ? empty_string() : h_preamble.str();
1233         if (!raw.empty()) {
1234                 os << "\\begin_preamble\n";
1235                 for (string::size_type i = 0; i < raw.size(); ++i) {
1236                         if (raw[i] == package_beg_sep) {
1237                                 // Here follows some package loading code that
1238                                 // must be skipped if the package is loaded
1239                                 // automatically.
1240                                 string::size_type j = raw.find(package_mid_sep, i);
1241                                 if (j == string::npos)
1242                                         return false;
1243                                 string::size_type k = raw.find(package_end_sep, j);
1244                                 if (k == string::npos)
1245                                         return false;
1246                                 string const package = raw.substr(i + 1, j - i - 1);
1247                                 string const replacement = raw.substr(j + 1, k - j - 1);
1248                                 if (auto_packages.find(package) == auto_packages.end())
1249                                         os << replacement;
1250                                 i = k;
1251                         } else
1252                                 os.put(raw[i]);
1253                 }
1254                 os << "\n\\end_preamble\n";
1255         }
1256         if (!h_options.empty())
1257                 os << "\\options " << h_options << "\n";
1258         os << "\\use_default_options " << h_use_default_options << "\n";
1259         if (!used_modules.empty()) {
1260                 os << "\\begin_modules\n";
1261                 vector<string>::const_iterator const end = used_modules.end();
1262                 vector<string>::const_iterator it = used_modules.begin();
1263                 for (; it != end; ++it)
1264                         os << *it << '\n';
1265                 os << "\\end_modules\n";
1266         }
1267         if (!h_includeonlys.empty()) {
1268                 os << "\\begin_includeonly\n";
1269                 for (auto const & iofile : h_includeonlys)
1270                         os << iofile << '\n';
1271                 os << "\\end_includeonly\n";
1272         }
1273         os << "\\maintain_unincluded_children " << h_maintain_unincluded_children << "\n"
1274            << "\\language " << h_language << "\n"
1275            << "\\language_package " << h_language_package << "\n"
1276            << "\\inputencoding " << h_inputencoding << "\n"
1277            << "\\fontencoding " << h_fontencoding << "\n"
1278            << "\\font_roman \"" << h_font_roman[0]
1279            << "\" \"" << h_font_roman[1] << "\"\n"
1280            << "\\font_sans \"" << h_font_sans[0] << "\" \"" << h_font_sans[1] << "\"\n"
1281            << "\\font_typewriter \"" << h_font_typewriter[0]
1282            << "\" \"" << h_font_typewriter[1] << "\"\n"
1283            << "\\font_math \"" << h_font_math[0] << "\" \"" << h_font_math[1] << "\"\n"
1284            << "\\font_default_family " << h_font_default_family << "\n"
1285            << "\\use_non_tex_fonts " << (h_use_non_tex_fonts ? "true" : "false") << '\n'
1286            << "\\font_sc " << h_font_sc << "\n"
1287            << "\\font_osf " << h_font_osf << "\n"
1288            << "\\font_sf_scale " << h_font_sf_scale[0]
1289            << ' ' << h_font_sf_scale[1] << '\n'
1290            << "\\font_tt_scale " << h_font_tt_scale[0]
1291            << ' ' << h_font_tt_scale[1] << '\n';
1292         if (!h_font_cjk.empty())
1293                 os << "\\font_cjk " << h_font_cjk << '\n';
1294         os << "\\use_microtype " << h_use_microtype << '\n'
1295            << "\\use_dash_ligatures " << h_use_dash_ligatures << '\n'
1296            << "\\graphics " << h_graphics << '\n'
1297            << "\\default_output_format " << h_default_output_format << "\n"
1298            << "\\output_sync " << h_output_sync << "\n";
1299         if (h_output_sync == "1")
1300                 os << "\\output_sync_macro \"" << h_output_sync_macro << "\"\n";
1301         os << "\\bibtex_command " << h_bibtex_command << "\n"
1302            << "\\index_command " << h_index_command << "\n";
1303         if (!h_float_placement.empty())
1304                 os << "\\float_placement " << h_float_placement << "\n";
1305         os << "\\paperfontsize " << h_paperfontsize << "\n"
1306            << "\\spacing " << h_spacing << "\n"
1307            << "\\use_hyperref " << h_use_hyperref << '\n';
1308         if (h_use_hyperref == "true") {
1309                 if (!h_pdf_title.empty())
1310                         os << "\\pdf_title " << Lexer::quoteString(h_pdf_title) << '\n';
1311                 if (!h_pdf_author.empty())
1312                         os << "\\pdf_author " << Lexer::quoteString(h_pdf_author) << '\n';
1313                 if (!h_pdf_subject.empty())
1314                         os << "\\pdf_subject " << Lexer::quoteString(h_pdf_subject) << '\n';
1315                 if (!h_pdf_keywords.empty())
1316                         os << "\\pdf_keywords " << Lexer::quoteString(h_pdf_keywords) << '\n';
1317                 os << "\\pdf_bookmarks " << h_pdf_bookmarks << "\n"
1318                       "\\pdf_bookmarksnumbered " << h_pdf_bookmarksnumbered << "\n"
1319                       "\\pdf_bookmarksopen " << h_pdf_bookmarksopen << "\n"
1320                       "\\pdf_bookmarksopenlevel " << h_pdf_bookmarksopenlevel << "\n"
1321                       "\\pdf_breaklinks " << h_pdf_breaklinks << "\n"
1322                       "\\pdf_pdfborder " << h_pdf_pdfborder << "\n"
1323                       "\\pdf_colorlinks " << h_pdf_colorlinks << "\n"
1324                       "\\pdf_backref " << h_pdf_backref << "\n"
1325                       "\\pdf_pdfusetitle " << h_pdf_pdfusetitle << '\n';
1326                 if (!h_pdf_pagemode.empty())
1327                         os << "\\pdf_pagemode " << h_pdf_pagemode << '\n';
1328                 if (!h_pdf_quoted_options.empty())
1329                         os << "\\pdf_quoted_options " << Lexer::quoteString(h_pdf_quoted_options) << '\n';
1330         }
1331         os << "\\papersize " << h_papersize << "\n"
1332            << "\\use_geometry " << h_use_geometry << '\n';
1333         for (map<string, string>::const_iterator it = h_use_packages.begin();
1334              it != h_use_packages.end(); ++it)
1335                 os << "\\use_package " << it->first << ' ' << it->second << '\n';
1336         os << "\\cite_engine " << h_cite_engine << '\n'
1337            << "\\cite_engine_type " << h_cite_engine_type << '\n'
1338            << "\\biblio_style " << h_biblio_style << "\n"
1339            << "\\use_bibtopic " << h_use_bibtopic << "\n";
1340         if (!h_biblio_options.empty())
1341                 os << "\\biblio_options " << h_biblio_options << "\n";
1342         if (!h_biblatex_bibstyle.empty())
1343                 os << "\\biblatex_bibstyle " << h_biblatex_bibstyle << "\n";
1344         if (!h_biblatex_citestyle.empty())
1345                 os << "\\biblatex_citestyle " << h_biblatex_citestyle << "\n";
1346         if (!h_multibib.empty())
1347                 os << "\\multibib " << h_multibib << "\n";
1348         os << "\\use_indices " << h_use_indices << "\n"
1349            << "\\paperorientation " << h_paperorientation << '\n'
1350            << "\\suppress_date " << h_suppress_date << '\n'
1351            << "\\justification " << h_justification << '\n'
1352            << "\\use_refstyle " << h_use_refstyle << '\n'
1353            << "\\use_minted " << h_use_minted << '\n';
1354         if (!h_fontcolor.empty())
1355                 os << "\\fontcolor " << h_fontcolor << '\n';
1356         if (!h_notefontcolor.empty())
1357                 os << "\\notefontcolor " << h_notefontcolor << '\n';
1358         if (!h_backgroundcolor.empty())
1359                 os << "\\backgroundcolor " << h_backgroundcolor << '\n';
1360         if (!h_boxbgcolor.empty())
1361                 os << "\\boxbgcolor " << h_boxbgcolor << '\n';
1362         if (index_number != 0)
1363                 for (int i = 0; i < index_number; i++) {
1364                         os << "\\index " << h_index[i] << '\n'
1365                            << "\\shortcut " << h_shortcut[i] << '\n'
1366                            << "\\color " << h_color << '\n'
1367                            << "\\end_index\n";
1368                 }
1369         else {
1370                 os << "\\index " << h_index[0] << '\n'
1371                    << "\\shortcut " << h_shortcut[0] << '\n'
1372                    << "\\color " << h_color << '\n'
1373                    << "\\end_index\n";
1374         }
1375         os << h_margins
1376            << "\\secnumdepth " << h_secnumdepth << "\n"
1377            << "\\tocdepth " << h_tocdepth << "\n"
1378            << "\\paragraph_separation " << h_paragraph_separation << "\n";
1379         if (h_paragraph_separation == "skip")
1380                 os << "\\defskip " << h_defskip << "\n";
1381         else
1382                 os << "\\paragraph_indentation " << h_paragraph_indentation << "\n";
1383         os << "\\is_math_indent " << h_is_mathindent << "\n";
1384         if (!h_mathindentation.empty())
1385                 os << "\\math_indentation " << h_mathindentation << "\n";
1386         os << "\\math_numbering_side " << h_math_numbering_side << "\n";
1387         os << "\\quotes_style " << h_quotes_style << "\n"
1388            << "\\dynamic_quotes " << h_dynamic_quotes << "\n"
1389            << "\\papercolumns " << h_papercolumns << "\n"
1390            << "\\papersides " << h_papersides << "\n"
1391            << "\\paperpagestyle " << h_paperpagestyle << "\n";
1392         if (!h_listings_params.empty())
1393                 os << "\\listings_params " << h_listings_params << "\n";
1394         os << "\\tracking_changes " << h_tracking_changes << "\n"
1395            << "\\output_changes " << h_output_changes << "\n"
1396            << "\\html_math_output " << h_html_math_output << "\n"
1397            << "\\html_css_as_file " << h_html_css_as_file << "\n"
1398            << "\\html_be_strict " << h_html_be_strict << "\n"
1399            << authors_
1400            << "\\end_header\n\n"
1401            << "\\begin_body\n";
1402         return true;
1403 }
1404
1405
1406 void Preamble::parse(Parser & p, string const & forceclass,
1407                      TeX2LyXDocClass & tc)
1408 {
1409         // initialize fixed types
1410         special_columns_['D'] = 3;
1411         parse(p, forceclass, false, tc);
1412 }
1413
1414
1415 void Preamble::parse(Parser & p, string const & forceclass,
1416                      bool detectEncoding, TeX2LyXDocClass & tc)
1417 {
1418         bool is_full_document = false;
1419         bool is_lyx_file = false;
1420         bool in_lyx_preamble = false;
1421
1422         // determine whether this is a full document or a fragment for inclusion
1423         while (p.good()) {
1424                 Token const & t = p.get_token();
1425
1426                 if (t.cat() == catEscape && t.cs() == "documentclass") {
1427                         is_full_document = true;
1428                         break;
1429                 }
1430         }
1431         p.reset();
1432
1433         if (detectEncoding && !is_full_document)
1434                 return;
1435
1436         while (is_full_document && p.good()) {
1437                 if (detectEncoding && h_inputencoding != "auto" &&
1438                     h_inputencoding != "default")
1439                         return;
1440
1441                 Token const & t = p.get_token();
1442
1443 #ifdef FILEDEBUG
1444                 if (!detectEncoding)
1445                         cerr << "t: " << t << '\n';
1446 #endif
1447
1448                 //
1449                 // cat codes
1450                 //
1451                 if (!in_lyx_preamble &&
1452                     (t.cat() == catLetter ||
1453                      t.cat() == catSuper ||
1454                      t.cat() == catSub ||
1455                      t.cat() == catOther ||
1456                      t.cat() == catMath ||
1457                      t.cat() == catActive ||
1458                      t.cat() == catBegin ||
1459                      t.cat() == catEnd ||
1460                      t.cat() == catAlign ||
1461                      t.cat() == catParameter)) {
1462                         h_preamble << t.cs();
1463                         continue;
1464                 }
1465
1466                 if (!in_lyx_preamble &&
1467                     (t.cat() == catSpace || t.cat() == catNewline)) {
1468                         h_preamble << t.asInput();
1469                         continue;
1470                 }
1471
1472                 if (t.cat() == catComment) {
1473                         static regex const islyxfile("%% LyX .* created this file");
1474                         static regex const usercommands("User specified LaTeX commands");
1475
1476                         string const comment = t.asInput();
1477
1478                         // magically switch encoding default if it looks like XeLaTeX
1479                         static string const magicXeLaTeX =
1480                                 "% This document must be compiled with XeLaTeX ";
1481                         if (comment.size() > magicXeLaTeX.size()
1482                                   && comment.substr(0, magicXeLaTeX.size()) == magicXeLaTeX
1483                                   && h_inputencoding == "auto") {
1484                                 if (!detectEncoding)
1485                                         cerr << "XeLaTeX comment found, switching to UTF8\n";
1486                                 h_inputencoding = "utf8";
1487                         }
1488                         smatch sub;
1489                         if (regex_search(comment, sub, islyxfile)) {
1490                                 is_lyx_file = true;
1491                                 in_lyx_preamble = true;
1492                         } else if (is_lyx_file
1493                                    && regex_search(comment, sub, usercommands))
1494                                 in_lyx_preamble = false;
1495                         else if (!in_lyx_preamble)
1496                                 h_preamble << t.asInput();
1497                         continue;
1498                 }
1499
1500                 if (t.cs() == "PassOptionsToPackage") {
1501                         string const poptions = p.getArg('{', '}');
1502                         string const package = p.verbatim_item();
1503                         extra_package_options_.insert(make_pair(package, poptions));
1504                         continue;
1505                 }
1506
1507                 if (t.cs() == "pagestyle") {
1508                         h_paperpagestyle = p.verbatim_item();
1509                         continue;
1510                 }
1511
1512                 if (t.cs() == "setdefaultlanguage") {
1513                         xetex = true;
1514                         // We don't yet care about non-language variant options
1515                         // because LyX doesn't support this yet, see bug #8214
1516                         if (p.hasOpt()) {
1517                                 string langopts = p.getOpt();
1518                                 // check if the option contains a variant, if yes, extract it
1519                                 string::size_type pos_var = langopts.find("variant");
1520                                 string::size_type i = langopts.find(',', pos_var);
1521                                 string::size_type k = langopts.find('=', pos_var);
1522                                 if (pos_var != string::npos){
1523                                         string variant;
1524                                         if (i == string::npos)
1525                                                 variant = langopts.substr(k + 1, langopts.length() - k - 2);
1526                                         else
1527                                                 variant = langopts.substr(k + 1, i - k - 1);
1528                                         h_language = variant;
1529                                 }
1530                                 p.verbatim_item();
1531                         } else
1532                                 h_language = p.verbatim_item();
1533                         //finally translate the poyglossia name to a LyX name
1534                         h_language = polyglossia2lyx(h_language);
1535                         continue;
1536                 }
1537
1538                 if (t.cs() == "setotherlanguage") {
1539                         // We don't yet care about the option because LyX doesn't
1540                         // support this yet, see bug #8214
1541                         p.hasOpt() ? p.getOpt() : string();
1542                         p.verbatim_item();
1543                         continue;
1544                 }
1545
1546                 if (t.cs() == "setmainfont") {
1547                         // we don't care about the option
1548                         p.hasOpt() ? p.getOpt() : string();
1549                         h_font_roman[1] = p.getArg('{', '}');
1550                         continue;
1551                 }
1552
1553                 if (t.cs() == "setsansfont" || t.cs() == "setmonofont") {
1554                         // LyX currently only supports the scale option
1555                         string scale;
1556                         if (p.hasOpt()) {
1557                                 string fontopts = p.getArg('[', ']');
1558                                 // check if the option contains a scaling, if yes, extract it
1559                                 string::size_type pos = fontopts.find("Scale");
1560                                 if (pos != string::npos) {
1561                                         string::size_type i = fontopts.find(',', pos);
1562                                         if (i == string::npos)
1563                                                 scale_as_percentage(fontopts.substr(pos + 1), scale);
1564                                         else
1565                                                 scale_as_percentage(fontopts.substr(pos, i - pos), scale);
1566                                 }
1567                         }
1568                         if (t.cs() == "setsansfont") {
1569                                 if (!scale.empty())
1570                                         h_font_sf_scale[1] = scale;
1571                                 h_font_sans[1] = p.getArg('{', '}');
1572                         } else {
1573                                 if (!scale.empty())
1574                                         h_font_tt_scale[1] = scale;
1575                                 h_font_typewriter[1] = p.getArg('{', '}');
1576                         }
1577                         continue;
1578                 }
1579
1580                 if (t.cs() == "date") {
1581                         string argument = p.getArg('{', '}');
1582                         if (argument.empty())
1583                                 h_suppress_date = "true";
1584                         else
1585                                 h_preamble << t.asInput() << '{' << argument << '}';
1586                         continue;
1587                 }
1588
1589                 if (t.cs() == "color") {
1590                         string const space =
1591                                 (p.hasOpt() ? p.getOpt() : string());
1592                         string argument = p.getArg('{', '}');
1593                         // check the case that a standard color is used
1594                         if (space.empty() && is_known(argument, known_basic_colors)) {
1595                                 h_fontcolor = rgbcolor2code(argument);
1596                                 registerAutomaticallyLoadedPackage("color");
1597                         } else if (space.empty() && argument == "document_fontcolor")
1598                                 registerAutomaticallyLoadedPackage("color");
1599                         // check the case that LyX's document_fontcolor is defined
1600                         // but not used for \color
1601                         else {
1602                                 h_preamble << t.asInput();
1603                                 if (!space.empty())
1604                                         h_preamble << space;
1605                                 h_preamble << '{' << argument << '}';
1606                                 // the color might already be set because \definecolor
1607                                 // is parsed before this
1608                                 h_fontcolor = "";
1609                         }
1610                         continue;
1611                 }
1612
1613                 if (t.cs() == "pagecolor") {
1614                         string argument = p.getArg('{', '}');
1615                         // check the case that a standard color is used
1616                         if (is_known(argument, known_basic_colors)) {
1617                                 h_backgroundcolor = rgbcolor2code(argument);
1618                         } else if (argument == "page_backgroundcolor")
1619                                 registerAutomaticallyLoadedPackage("color");
1620                         // check the case that LyX's page_backgroundcolor is defined
1621                         // but not used for \pagecolor
1622                         else {
1623                                 h_preamble << t.asInput() << '{' << argument << '}';
1624                                 // the color might already be set because \definecolor
1625                                 // is parsed before this
1626                                 h_backgroundcolor = "";
1627                         }
1628                         continue;
1629                 }
1630
1631                 if (t.cs() == "makeatletter") {
1632                         // LyX takes care of this
1633                         p.setCatcode('@', catLetter);
1634                         continue;
1635                 }
1636
1637                 if (t.cs() == "makeatother") {
1638                         // LyX takes care of this
1639                         p.setCatcode('@', catOther);
1640                         continue;
1641                 }
1642
1643                 if (t.cs() == "makeindex") {
1644                         // LyX will re-add this if a print index command is found
1645                         p.skip_spaces();
1646                         continue;
1647                 }
1648
1649                 if (t.cs() == "newindex") {
1650                         string const indexname = p.getArg('[', ']');
1651                         string const shortcut = p.verbatim_item();
1652                         if (!indexname.empty())
1653                                 h_index[index_number] = indexname;
1654                         else
1655                                 h_index[index_number] = shortcut;
1656                         h_shortcut[index_number] = shortcut;
1657                         index_number += 1;
1658                         p.skip_spaces();
1659                         continue;
1660                 }
1661
1662                 if (t.cs() == "addbibresource") {
1663                         biblatex_bibliographies.push_back(removeExtension(p.getArg('{', '}')));
1664                         continue;
1665                 }
1666
1667                 if (t.cs() == "bibliography") {
1668                         vector<string> bibs = getVectorFromString(p.getArg('{', '}'));
1669                         biblatex_bibliographies.insert(biblatex_bibliographies.end(), bibs.begin(), bibs.end());
1670                         continue;
1671                 }
1672
1673                 if (t.cs() == "RS@ifundefined") {
1674                         string const name = p.verbatim_item();
1675                         string const body1 = p.verbatim_item();
1676                         string const body2 = p.verbatim_item();
1677                         // only non-lyxspecific stuff
1678                         if (in_lyx_preamble &&
1679                             (name == "subsecref" || name == "thmref" || name == "lemref"))
1680                                 p.skip_spaces();
1681                         else {
1682                                 ostringstream ss;
1683                                 ss << '\\' << t.cs();
1684                                 ss << '{' << name << '}'
1685                                    << '{' << body1 << '}'
1686                                    << '{' << body2 << '}';
1687                                 h_preamble << ss.str();
1688                         }
1689                         continue;
1690                 }
1691
1692                 if (t.cs() == "AtBeginDocument") {
1693                         string const name = p.verbatim_item();
1694                         // only non-lyxspecific stuff
1695                         if (in_lyx_preamble &&
1696                             (name == "\\providecommand\\partref[1]{\\ref{part:#1}}"
1697                                 || name == "\\providecommand\\chapref[1]{\\ref{chap:#1}}"
1698                                 || name == "\\providecommand\\secref[1]{\\ref{sec:#1}}"
1699                                 || name == "\\providecommand\\subsecref[1]{\\ref{subsec:#1}}"
1700                                 || name == "\\providecommand\\parref[1]{\\ref{par:#1}}"
1701                                 || name == "\\providecommand\\figref[1]{\\ref{fig:#1}}"
1702                                 || name == "\\providecommand\\tabref[1]{\\ref{tab:#1}}"
1703                                 || name == "\\providecommand\\algref[1]{\\ref{alg:#1}}"
1704                                 || name == "\\providecommand\\fnref[1]{\\ref{fn:#1}}"
1705                                 || name == "\\providecommand\\enuref[1]{\\ref{enu:#1}}"
1706                                 || name == "\\providecommand\\eqref[1]{\\ref{eq:#1}}"
1707                                 || name == "\\providecommand\\lemref[1]{\\ref{lem:#1}}"
1708                                 || name == "\\providecommand\\thmref[1]{\\ref{thm:#1}}"
1709                                 || name == "\\providecommand\\corref[1]{\\ref{cor:#1}}"
1710                                 || name == "\\providecommand\\propref[1]{\\ref{prop:#1}}"))
1711                                 p.skip_spaces();
1712                         else {
1713                                 ostringstream ss;
1714                                 ss << '\\' << t.cs();
1715                                 ss << '{' << name << '}';
1716                                 h_preamble << ss.str();
1717                         }
1718                         continue;
1719                 }
1720
1721                 if (t.cs() == "newcommand" || t.cs() == "newcommandx"
1722                     || t.cs() == "renewcommand" || t.cs() == "renewcommandx"
1723                     || t.cs() == "providecommand" || t.cs() == "providecommandx"
1724                     || t.cs() == "DeclareRobustCommand"
1725                     || t.cs() == "DeclareRobustCommandx"
1726                     || t.cs() == "ProvideTextCommandDefault"
1727                     || t.cs() == "DeclareMathAccent") {
1728                         bool star = false;
1729                         if (p.next_token().character() == '*') {
1730                                 p.get_token();
1731                                 star = true;
1732                         }
1733                         string const name = p.verbatim_item();
1734                         string const opt1 = p.getFullOpt();
1735                         string const opt2 = p.getFullOpt();
1736                         string const body = p.verbatim_item();
1737                         // store the in_lyx_preamble setting
1738                         bool const was_in_lyx_preamble = in_lyx_preamble;
1739                         // font settings
1740                         if (name == "\\rmdefault")
1741                                 if (is_known(body, known_roman_fonts)) {
1742                                         h_font_roman[0] = body;
1743                                         p.skip_spaces();
1744                                         in_lyx_preamble = true;
1745                                 }
1746                         if (name == "\\sfdefault")
1747                                 if (is_known(body, known_sans_fonts)) {
1748                                         h_font_sans[0] = body;
1749                                         p.skip_spaces();
1750                                         in_lyx_preamble = true;
1751                                 }
1752                         if (name == "\\ttdefault")
1753                                 if (is_known(body, known_typewriter_fonts)) {
1754                                         h_font_typewriter[0] = body;
1755                                         p.skip_spaces();
1756                                         in_lyx_preamble = true;
1757                                 }
1758                         if (name == "\\familydefault") {
1759                                 string family = body;
1760                                 // remove leading "\"
1761                                 h_font_default_family = family.erase(0,1);
1762                                 p.skip_spaces();
1763                                 in_lyx_preamble = true;
1764                         }
1765
1766                         // remove LyX-specific definitions that are re-added by LyX
1767                         // if necessary
1768                         // \lyxline is an ancient command that is converted by tex2lyx into
1769                         // a \rule therefore remove its preamble code
1770                         if (name == "\\lyxdot" || name == "\\lyxarrow"
1771                             || name == "\\lyxline" || name == "\\LyX") {
1772                                 p.skip_spaces();
1773                                 in_lyx_preamble = true;
1774                         }
1775
1776                         // Add the command to the known commands
1777                         add_known_command(name, opt1, !opt2.empty(), from_utf8(body));
1778
1779                         // only non-lyxspecific stuff
1780                         if (!in_lyx_preamble) {
1781                                 ostringstream ss;
1782                                 ss << '\\' << t.cs();
1783                                 if (star)
1784                                         ss << '*';
1785                                 ss << '{' << name << '}' << opt1 << opt2
1786                                    << '{' << body << "}";
1787                                 if (prefixIs(t.cs(), "renew") || !contains(h_preamble.str(), ss.str()))
1788                                         h_preamble << ss.str();
1789 /*
1790                                 ostream & out = in_preamble ? h_preamble : os;
1791                                 out << "\\" << t.cs() << "{" << name << "}"
1792                                     << opts << "{" << body << "}";
1793 */
1794                         }
1795                         // restore the in_lyx_preamble setting
1796                         in_lyx_preamble = was_in_lyx_preamble;
1797                         continue;
1798                 }
1799
1800                 if (t.cs() == "documentclass") {
1801                         vector<string>::iterator it;
1802                         vector<string> opts = split_options(p.getArg('[', ']'));
1803                         handle_opt(opts, known_fontsizes, h_paperfontsize);
1804                         delete_opt(opts, known_fontsizes);
1805                         // delete "pt" at the end
1806                         string::size_type i = h_paperfontsize.find("pt");
1807                         if (i != string::npos)
1808                                 h_paperfontsize.erase(i);
1809                         // The documentclass options are always parsed before the options
1810                         // of the babel call so that a language cannot overwrite the babel
1811                         // options.
1812                         handle_opt(opts, known_languages, h_language);
1813                         delete_opt(opts, known_languages);
1814
1815                         // math indentation
1816                         if ((it = find(opts.begin(), opts.end(), "fleqn"))
1817                                  != opts.end()) {
1818                                 h_is_mathindent = "1";
1819                                 opts.erase(it);
1820                         }
1821                         // formula numbering side
1822                         if ((it = find(opts.begin(), opts.end(), "leqno"))
1823                                  != opts.end()) {
1824                                 h_math_numbering_side = "left";
1825                                 opts.erase(it);
1826                         }
1827                         else if ((it = find(opts.begin(), opts.end(), "reqno"))
1828                                  != opts.end()) {
1829                                 h_math_numbering_side = "right";
1830                                 opts.erase(it);
1831                         }
1832
1833                         // paper orientation
1834                         if ((it = find(opts.begin(), opts.end(), "landscape")) != opts.end()) {
1835                                 h_paperorientation = "landscape";
1836                                 opts.erase(it);
1837                         }
1838                         // paper sides
1839                         if ((it = find(opts.begin(), opts.end(), "oneside"))
1840                                  != opts.end()) {
1841                                 h_papersides = "1";
1842                                 opts.erase(it);
1843                         }
1844                         if ((it = find(opts.begin(), opts.end(), "twoside"))
1845                                  != opts.end()) {
1846                                 h_papersides = "2";
1847                                 opts.erase(it);
1848                         }
1849                         // paper columns
1850                         if ((it = find(opts.begin(), opts.end(), "onecolumn"))
1851                                  != opts.end()) {
1852                                 h_papercolumns = "1";
1853                                 opts.erase(it);
1854                         }
1855                         if ((it = find(opts.begin(), opts.end(), "twocolumn"))
1856                                  != opts.end()) {
1857                                 h_papercolumns = "2";
1858                                 opts.erase(it);
1859                         }
1860                         // paper sizes
1861                         // some size options are known to any document classes, other sizes
1862                         // are handled by the \geometry command of the geometry package
1863                         handle_opt(opts, known_class_paper_sizes, h_papersize);
1864                         delete_opt(opts, known_class_paper_sizes);
1865                         // the remaining options
1866                         h_options = join(opts, ",");
1867                         // FIXME This does not work for classes that have a
1868                         //       different name in LyX than in LaTeX
1869                         h_textclass = p.getArg('{', '}');
1870                         p.skip_spaces();
1871                         continue;
1872                 }
1873
1874                 if (t.cs() == "usepackage") {
1875                         string const options = p.getArg('[', ']');
1876                         string const name = p.getArg('{', '}');
1877                         vector<string> vecnames;
1878                         split(name, vecnames, ',');
1879                         vector<string>::const_iterator it  = vecnames.begin();
1880                         vector<string>::const_iterator end = vecnames.end();
1881                         for (; it != end; ++it)
1882                                 handle_package(p, trimSpaceAndEol(*it), options,
1883                                                in_lyx_preamble, detectEncoding);
1884                         continue;
1885                 }
1886
1887                 if (t.cs() == "inputencoding") {
1888                         string const encoding = p.getArg('{','}');
1889                         Encoding const * const enc = encodings.fromLaTeXName(
1890                                 encoding, Encoding::inputenc, true);
1891                         if (!enc) {
1892                                 if (!detectEncoding)
1893                                         cerr << "Unknown encoding " << encoding
1894                                              << ". Ignoring." << std::endl;
1895                         } else {
1896                                 if (!enc->unsafe())
1897                                         h_inputencoding = enc->name();
1898                                 p.setEncoding(enc->iconvName());
1899                         }
1900                         continue;
1901                 }
1902
1903                 if (t.cs() == "newenvironment") {
1904                         string const name = p.getArg('{', '}');
1905                         string const opt1 = p.getFullOpt();
1906                         string const opt2 = p.getFullOpt();
1907                         string const beg = p.verbatim_item();
1908                         string const end = p.verbatim_item();
1909                         if (!in_lyx_preamble) {
1910                                 h_preamble << "\\newenvironment{" << name
1911                                            << '}' << opt1 << opt2 << '{'
1912                                            << beg << "}{" << end << '}';
1913                         }
1914                         add_known_environment(name, opt1, !opt2.empty(),
1915                                               from_utf8(beg), from_utf8(end));
1916                         continue;
1917                 }
1918
1919                 if (t.cs() == "newtheorem") {
1920                         bool star = false;
1921                         if (p.next_token().character() == '*') {
1922                                 p.get_token();
1923                                 star = true;
1924                         }
1925                         string const name = p.getArg('{', '}');
1926                         string const opt1 = p.getFullOpt();
1927                         string const opt2 = p.getFullOpt();
1928                         string const body = p.verbatim_item();
1929                         string const opt3 = p.getFullOpt();
1930                         string const cmd = star ? "\\newtheorem*" : "\\newtheorem";
1931
1932                         string const complete = cmd + "{" + name + '}' +
1933                                           opt1 + opt2 + '{' + body + '}' + opt3;
1934
1935                         add_known_theorem(name, opt1, !opt2.empty(), from_utf8(complete));
1936
1937                         if (!in_lyx_preamble)
1938                                 h_preamble << complete;
1939                         continue;
1940                 }
1941
1942                 if (t.cs() == "def") {
1943                         string name = p.get_token().cs();
1944                         // In fact, name may be more than the name:
1945                         // In the test case of bug 8116
1946                         // name == "csname SF@gobble@opt \endcsname".
1947                         // Therefore, we need to use asInput() instead of cs().
1948                         while (p.next_token().cat() != catBegin)
1949                                 name += p.get_token().asInput();
1950                         if (!in_lyx_preamble)
1951                                 h_preamble << "\\def\\" << name << '{'
1952                                            << p.verbatim_item() << "}";
1953                         continue;
1954                 }
1955
1956                 if (t.cs() == "newcolumntype") {
1957                         string const name = p.getArg('{', '}');
1958                         trimSpaceAndEol(name);
1959                         int nargs = 0;
1960                         string opts = p.getOpt();
1961                         if (!opts.empty()) {
1962                                 istringstream is(string(opts, 1));
1963                                 is >> nargs;
1964                         }
1965                         special_columns_[name[0]] = nargs;
1966                         h_preamble << "\\newcolumntype{" << name << "}";
1967                         if (nargs)
1968                                 h_preamble << "[" << nargs << "]";
1969                         h_preamble << "{" << p.verbatim_item() << "}";
1970                         continue;
1971                 }
1972
1973                 if (t.cs() == "setcounter") {
1974                         string const name = p.getArg('{', '}');
1975                         string const content = p.getArg('{', '}');
1976                         if (name == "secnumdepth")
1977                                 h_secnumdepth = content;
1978                         else if (name == "tocdepth")
1979                                 h_tocdepth = content;
1980                         else
1981                                 h_preamble << "\\setcounter{" << name << "}{" << content << "}";
1982                         continue;
1983                 }
1984
1985                 if (t.cs() == "setlength") {
1986                         string const name = p.verbatim_item();
1987                         string const content = p.verbatim_item();
1988                         // the paragraphs are only not indented when \parindent is set to zero
1989                         if (name == "\\parindent" && content != "") {
1990                                 if (content[0] == '0')
1991                                         h_paragraph_separation = "skip";
1992                                 else
1993                                         h_paragraph_indentation = translate_len(content);
1994                         } else if (name == "\\parskip") {
1995                                 if (content == "\\smallskipamount")
1996                                         h_defskip = "smallskip";
1997                                 else if (content == "\\medskipamount")
1998                                         h_defskip = "medskip";
1999                                 else if (content == "\\bigskipamount")
2000                                         h_defskip = "bigskip";
2001                                 else
2002                                         h_defskip = translate_len(content);
2003                         } else if (name == "\\mathindent") {
2004                                 h_mathindentation = translate_len(content);
2005                         } else
2006                                 h_preamble << "\\setlength{" << name << "}{" << content << "}";
2007                         continue;
2008                 }
2009
2010                 if (t.cs() == "onehalfspacing") {
2011                         h_spacing = "onehalf";
2012                         continue;
2013                 }
2014
2015                 if (t.cs() == "doublespacing") {
2016                         h_spacing = "double";
2017                         continue;
2018                 }
2019
2020                 if (t.cs() == "setstretch") {
2021                         h_spacing = "other " + p.verbatim_item();
2022                         continue;
2023                 }
2024
2025                 if (t.cs() == "synctex") {
2026                         // the scheme is \synctex=value
2027                         // where value can only be "1" or "-1"
2028                         h_output_sync = "1";
2029                         // there can be any character behind the value (e.g. a linebreak or a '\'
2030                         // therefore we extract it char by char
2031                         p.get_token();
2032                         string value = p.get_token().asInput();
2033                         if (value == "-")
2034                                 value += p.get_token().asInput();
2035                         h_output_sync_macro = "\\synctex=" + value;
2036                         continue;
2037                 }
2038
2039                 if (t.cs() == "begin") {
2040                         string const name = p.getArg('{', '}');
2041                         if (name == "document")
2042                                 break;
2043                         h_preamble << "\\begin{" << name << "}";
2044                         continue;
2045                 }
2046
2047                 if (t.cs() == "geometry") {
2048                         vector<string> opts = split_options(p.getArg('{', '}'));
2049                         handle_geometry(opts);
2050                         continue;
2051                 }
2052
2053                 if (t.cs() == "definecolor") {
2054                         string const color = p.getArg('{', '}');
2055                         string const space = p.getArg('{', '}');
2056                         string const value = p.getArg('{', '}');
2057                         if (color == "document_fontcolor" && space == "rgb") {
2058                                 RGBColor c(RGBColorFromLaTeX(value));
2059                                 h_fontcolor = X11hexname(c);
2060                         } else if (color == "note_fontcolor" && space == "rgb") {
2061                                 RGBColor c(RGBColorFromLaTeX(value));
2062                                 h_notefontcolor = X11hexname(c);
2063                         } else if (color == "page_backgroundcolor" && space == "rgb") {
2064                                 RGBColor c(RGBColorFromLaTeX(value));
2065                                 h_backgroundcolor = X11hexname(c);
2066                         } else if (color == "shadecolor" && space == "rgb") {
2067                                 RGBColor c(RGBColorFromLaTeX(value));
2068                                 h_boxbgcolor = X11hexname(c);
2069                         } else {
2070                                 h_preamble << "\\definecolor{" << color
2071                                            << "}{" << space << "}{" << value
2072                                            << '}';
2073                         }
2074                         continue;
2075                 }
2076
2077                 if (t.cs() == "bibliographystyle") {
2078                         h_biblio_style = p.verbatim_item();
2079                         continue;
2080                 }
2081
2082                 if (t.cs() == "jurabibsetup") {
2083                         // FIXME p.getArg('{', '}') is most probably wrong (it
2084                         //       does not handle nested braces).
2085                         //       Use p.verbatim_item() instead.
2086                         vector<string> jurabibsetup =
2087                                 split_options(p.getArg('{', '}'));
2088                         // add jurabibsetup to the jurabib package options
2089                         add_package("jurabib", jurabibsetup);
2090                         if (!jurabibsetup.empty()) {
2091                                 h_preamble << "\\jurabibsetup{"
2092                                            << join(jurabibsetup, ",") << '}';
2093                         }
2094                         continue;
2095                 }
2096
2097                 if (t.cs() == "hypersetup") {
2098                         vector<string> hypersetup =
2099                                 split_options(p.verbatim_item());
2100                         // add hypersetup to the hyperref package options
2101                         handle_hyperref(hypersetup);
2102                         if (!hypersetup.empty()) {
2103                                 h_preamble << "\\hypersetup{"
2104                                            << join(hypersetup, ",") << '}';
2105                         }
2106                         continue;
2107                 }
2108
2109                 if (t.cs() == "includeonly") {
2110                         vector<string> includeonlys = getVectorFromString(p.getArg('{', '}'));
2111                         for (auto & iofile : includeonlys) {
2112                                 string filename(normalize_filename(iofile));
2113                                 string const path = getMasterFilePath(true);
2114                                 // We want to preserve relative/absolute filenames,
2115                                 // therefore path is only used for testing
2116                                 if (!makeAbsPath(filename, path).exists()) {
2117                                         // The file extension is probably missing.
2118                                         // Now try to find it out.
2119                                         string const tex_name =
2120                                                 find_file(filename, path,
2121                                                           known_tex_extensions);
2122                                         if (!tex_name.empty())
2123                                                 filename = tex_name;
2124                                 }
2125                                 string outname;
2126                                 if (makeAbsPath(filename, path).exists())
2127                                         fix_child_filename(filename);
2128                                 else
2129                                         cerr << "Warning: Could not find included file '"
2130                                              << filename << "'." << endl;
2131                                 outname = changeExtension(filename, "lyx");
2132                                 h_includeonlys.push_back(outname);
2133                         }
2134                         continue;
2135                 }
2136
2137                 if (is_known(t.cs(), known_if_3arg_commands)) {
2138                         // prevent misparsing of \usepackage if it is used
2139                         // as an argument (see e.g. our own output of
2140                         // \@ifundefined above)
2141                         string const arg1 = p.verbatim_item();
2142                         string const arg2 = p.verbatim_item();
2143                         string const arg3 = p.verbatim_item();
2144                         // test case \@ifundefined{date}{}{\date{}}
2145                         if (t.cs() == "@ifundefined" && arg1 == "date" &&
2146                             arg2.empty() && arg3 == "\\date{}") {
2147                                 h_suppress_date = "true";
2148                         // older tex2lyx versions did output
2149                         // \@ifundefined{definecolor}{\usepackage{color}}{}
2150                         } else if (t.cs() == "@ifundefined" &&
2151                                    arg1 == "definecolor" &&
2152                                    arg2 == "\\usepackage{color}" &&
2153                                    arg3.empty()) {
2154                                 if (!in_lyx_preamble)
2155                                         h_preamble << package_beg_sep
2156                                                    << "color"
2157                                                    << package_mid_sep
2158                                                    << "\\@ifundefined{definecolor}{color}{}"
2159                                                    << package_end_sep;
2160                         // test for case
2161                         //\@ifundefined{showcaptionsetup}{}{%
2162                         // \PassOptionsToPackage{caption=false}{subfig}}
2163                         // that LyX uses for subfloats
2164                         } else if (t.cs() == "@ifundefined" &&
2165                                    arg1 == "showcaptionsetup" && arg2.empty()
2166                                 && arg3 == "%\n \\PassOptionsToPackage{caption=false}{subfig}") {
2167                                 ; // do nothing
2168                         } else if (!in_lyx_preamble) {
2169                                 h_preamble << t.asInput()
2170                                            << '{' << arg1 << '}'
2171                                            << '{' << arg2 << '}'
2172                                            << '{' << arg3 << '}';
2173                         }
2174                         continue;
2175                 }
2176
2177                 if (is_known(t.cs(), known_if_commands)) {
2178                         // must not parse anything in conditional code, since
2179                         // LyX would output the parsed contents unconditionally
2180                         if (!in_lyx_preamble)
2181                                 h_preamble << t.asInput();
2182                         handle_if(p, in_lyx_preamble);
2183                         continue;
2184                 }
2185
2186                 if (!t.cs().empty() && !in_lyx_preamble) {
2187                         h_preamble << '\\' << t.cs();
2188                         continue;
2189                 }
2190         }
2191
2192         // remove the whitespace
2193         p.skip_spaces();
2194
2195         // Force textclass if the user wanted it
2196         if (!forceclass.empty())
2197                 h_textclass = forceclass;
2198         tc.setName(h_textclass);
2199         if (!LayoutFileList::get().haveClass(h_textclass) || !tc.load()) {
2200                 cerr << "Error: Could not read layout file for textclass \"" << h_textclass << "\"." << endl;
2201                 exit(EXIT_FAILURE);
2202         }
2203         if (h_papersides.empty()) {
2204                 ostringstream ss;
2205                 ss << tc.sides();
2206                 h_papersides = ss.str();
2207         }
2208
2209         // If the CJK package is used we cannot set the document language from
2210         // the babel options. Instead, we guess which language is used most
2211         // and set this one.
2212         default_language = h_language;
2213         if (is_full_document &&
2214             (auto_packages.find("CJK") != auto_packages.end() ||
2215              auto_packages.find("CJKutf8") != auto_packages.end())) {
2216                 p.pushPosition();
2217                 h_language = guessLanguage(p, default_language);
2218                 p.popPosition();
2219                 if (explicit_babel && h_language != default_language) {
2220                         // We set the document language to a CJK language,
2221                         // but babel is explicitly called in the user preamble
2222                         // without options. LyX will not add the default
2223                         // language to the document options if it is either
2224                         // english, or no text is set as default language.
2225                         // Therefore we need to add a language option explicitly.
2226                         // FIXME: It would be better to remove all babel calls
2227                         //        from the user preamble, but this is difficult
2228                         //        without re-introducing bug 7861.
2229                         if (h_options.empty())
2230                                 h_options = lyx2babel(default_language);
2231                         else
2232                                 h_options += ',' + lyx2babel(default_language);
2233                 }
2234         }
2235
2236         // Finally, set the quote style.
2237         // LyX knows the following quotes styles:
2238         // british, cjk, cjkangle, danish, english, french, german,
2239         // polish, russian, swedish and swiss
2240         // conversion list taken from
2241         // https://en.wikipedia.org/wiki/Quotation_mark,_non-English_usage
2242         // (quotes for kazakh are unknown)
2243         // british
2244         if (is_known(h_language, known_british_quotes_languages))
2245                 h_quotes_style = "british";
2246         // cjk
2247         else if (is_known(h_language, known_cjk_quotes_languages))
2248                 h_quotes_style = "cjk";
2249         // cjkangle
2250         else if (is_known(h_language, known_cjkangle_quotes_languages))
2251                 h_quotes_style = "cjkangle";
2252         // danish
2253         else if (is_known(h_language, known_danish_quotes_languages))
2254                 h_quotes_style = "danish";
2255         // french
2256         else if (is_known(h_language, known_french_quotes_languages))
2257                 h_quotes_style = "french";
2258         // german
2259         else if (is_known(h_language, known_german_quotes_languages))
2260                 h_quotes_style = "german";
2261         // polish
2262         else if (is_known(h_language, known_polish_quotes_languages))
2263                 h_quotes_style = "polish";
2264         // russian
2265         else if (is_known(h_language, known_russian_quotes_languages))
2266                 h_quotes_style = "russian";
2267         // swedish
2268         else if (is_known(h_language, known_swedish_quotes_languages))
2269                 h_quotes_style = "swedish";
2270         // swiss
2271         else if (is_known(h_language, known_swiss_quotes_languages))
2272                 h_quotes_style = "swiss";
2273         // english
2274         else if (is_known(h_language, known_english_quotes_languages))
2275                 h_quotes_style = "english";
2276 }
2277
2278
2279 string Preamble::parseEncoding(Parser & p, string const & forceclass)
2280 {
2281         TeX2LyXDocClass dummy;
2282         parse(p, forceclass, true, dummy);
2283         if (h_inputencoding != "auto" && h_inputencoding != "default")
2284                 return h_inputencoding;
2285         return "";
2286 }
2287
2288
2289 string babel2lyx(string const & language)
2290 {
2291         char const * const * where = is_known(language, known_languages);
2292         if (where)
2293                 return known_coded_languages[where - known_languages];
2294         return language;
2295 }
2296
2297
2298 string lyx2babel(string const & language)
2299 {
2300         char const * const * where = is_known(language, known_coded_languages);
2301         if (where)
2302                 return known_languages[where - known_coded_languages];
2303         return language;
2304 }
2305
2306
2307 string Preamble::polyglossia2lyx(string const & language)
2308 {
2309         char const * const * where = is_known(language, polyglossia_languages);
2310         if (where)
2311                 return coded_polyglossia_languages[where - polyglossia_languages];
2312         return language;
2313 }
2314
2315
2316 string rgbcolor2code(string const & name)
2317 {
2318         char const * const * where = is_known(name, known_basic_colors);
2319         if (where) {
2320                 // "red", "green" etc
2321                 return known_basic_color_codes[where - known_basic_colors];
2322         }
2323         // "255,0,0", "0,255,0" etc
2324         RGBColor c(RGBColorFromLaTeX(name));
2325         return X11hexname(c);
2326 }
2327
2328 // }])
2329
2330
2331 } // namespace lyx