src/tex2lyx/Preamble.cpp

   1 /**
   2  * \file Preamble.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Uwe Stöhr
   8  *
   9  * Full author contact details are available in file CREDITS.
  10  */
  11
  12 // {[(
  13
  14 #include <config.h>
  15
  16 #include "Preamble.h"
  17 #include "tex2lyx.h"
  18
  19 #include "LayoutFile.h"
  20 #include "Layout.h"
  21 #include "Lexer.h"
  22 #include "TextClass.h"
  23
  24 #include "support/convert.h"
  25 #include "support/FileName.h"
  26 #include "support/filetools.h"
  27 #include "support/lstrings.h"
  28
  29 #include "support/regex.h"
  30
  31 #include <algorithm>
  32 #include <iostream>
  33
  34 using namespace std;
  35 using namespace lyx::support;
  36
  37
  38 namespace lyx {
  39
  40 // special columntypes
  41 extern map<char, int> special_columns;
  42
  43 Preamble preamble;
  44
  45 namespace {
  46
  47 // CJK languages are handled in text.cpp, polyglossia languages are listed
  48 // further down.
  49 /**
  50  * known babel language names (including synonyms)
  51  * not in standard babel: arabic, arabtex, armenian, belarusian, serbian-latin, thai
  52  * please keep this in sync with known_coded_languages line by line!
  53  */
  54 const char * const known_languages[] = {"acadian", "afrikaans", "albanian",
  55 "american", "arabic", "arabtex", "australian", "austrian", "bahasa", "bahasai",
  56 "bahasam", "basque", "belarusian", "brazil", "brazilian", "breton", "british",
  57 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
  58 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "francais",
  59 "french", "frenchb", "frenchle", "frenchpro", "galician", "german", "germanb",
  60 "greek", "hebrew", "hungarian", "icelandic", "indon", "indonesian", "interlingua",
  61 "irish", "italian", "japanese", "kazakh", "kurmanji", "latin", "latvian", "lithuanian",
  62 "lowersorbian", "lsorbian", "magyar", "malay", "meyalu", "mongolian", "naustrian",
  63 "newzealand", "ngerman", "ngermanb", "norsk", "nynorsk", "polutonikogreek", "polish",
  64 "portuges", "portuguese", "romanian", "russian", "russianb", "samin",
  65 "scottish", "serbian", "serbian-latin", "slovak", "slovene", "spanish",
  66 "swedish", "thai", "turkish", "turkmen", "ukraineb", "ukrainian",
  67 "uppersorbian", "UKenglish", "USenglish", "usorbian", "vietnam", "welsh",
  68 0};
  69
  70 /**
  71  * the same as known_languages with .lyx names
  72  * please keep this in sync with known_languages line by line!
  73  */
  74 const char * const known_coded_languages[] = {"french", "afrikaans", "albanian",
  75 "american", "arabic_arabi", "arabic_arabtex", "australian", "austrian", "bahasa", "bahasa",
  76 "bahasam", "basque", "belarusian", "brazilian", "brazilian", "breton", "british",
  77 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
  78 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "french",
  79 "french", "french", "french", "french", "galician", "german", "german",
  80 "greek", "hebrew", "magyar", "icelandic", "bahasa", "bahasa", "interlingua",
  81 "irish", "italian", "japanese", "kazakh", "kurmanji", "latin", "latvian", "lithuanian",
  82 "lowersorbian", "lowersorbian", "magyar", "bahasam", "bahasam", "mongolian", "naustrian",
  83 "newzealand", "ngerman", "ngerman", "norsk", "nynorsk", "polutonikogreek", "polish",
  84 "portuguese", "portuguese", "romanian", "russian", "russian", "samin",
  85 "scottish", "serbian", "serbian-latin", "slovak", "slovene", "spanish",
  86 "swedish", "thai", "turkish", "turkmen", "ukrainian", "ukrainian",
  87 "uppersorbian", "uppersorbian", "english", "english", "vietnamese", "welsh",
  88 0};
  89
  90 /// languages with danish quotes (.lyx names)
  91 const char * const known_danish_quotes_languages[] = {"danish", 0};
  92
  93 /// languages with english quotes (.lyx names)
  94 const char * const known_english_quotes_languages[] = {"american", "australian",
  95 "bahasa", "bahasam", "brazilian", "canadian", "chinese-simplified", "english",
  96 "esperanto", "hebrew", "irish", "korean", "newzealand", "portuguese", "scottish",
  97 "thai", 0};
  98
  99 /// languages with french quotes (.lyx names)
 100 const char * const known_french_quotes_languages[] = {"albanian",
 101 "arabic_arabi", "arabic_arabtex", "basque", "canadien", "catalan", "french",
 102 "galician", "greek", "italian", "norsk", "nynorsk", "polutonikogreek",
 103 "russian", "spanish", "spanish-mexico", "turkish", "turkmen", "ukrainian",
 104 "vietnamese", 0};
 105
 106 /// languages with german quotes (.lyx names)
 107 const char * const known_german_quotes_languages[] = {"austrian", "bulgarian",
 108 "czech", "german", "icelandic", "lithuanian", "lowersorbian", "naustrian",
 109 "ngerman", "serbian", "serbian-latin", "slovak", "slovene", "uppersorbian", 0};
 110
 111 /// languages with polish quotes (.lyx names)
 112 const char * const known_polish_quotes_languages[] = {"afrikaans", "croatian",
 113 "dutch", "estonian", "magyar", "polish", "romanian", 0};
 114
 115 /// languages with swedish quotes (.lyx names)
 116 const char * const known_swedish_quotes_languages[] = {"finnish",
 117 "swedish", 0};
 118
 119 /// known language packages from the times before babel
 120 const char * const known_old_language_packages[] = {"french", "frenchle",
 121 "frenchpro", "german", "ngerman", "pmfrench", 0};
 122
 123 char const * const known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
 124
 125 const char * const known_roman_fonts[] = { "ae", "beraserif", "bookman",
 126 "ccfonts", "chancery", "charter", "cmr", "fourier", "lmodern", "mathpazo",
 127 "mathptmx", "newcent", "utopia", 0};
 128
 129 const char * const known_sans_fonts[] = { "avant", "berasans", "cmbr", "cmss",
 130 "helvet", "lmss", 0};
 131
 132 const char * const known_typewriter_fonts[] = { "beramono", "cmtl", "cmtt",
 133 "courier", "lmtt", "luximono", "fourier", "lmodern", "mathpazo", "mathptmx",
 134 "newcent", 0};
 135
 136 const char * const known_paper_sizes[] = { "a0paper", "b0paper", "c0paper",
 137 "a1paper", "b1paper", "c1paper", "a2paper", "b2paper", "c2paper", "a3paper",
 138 "b3paper", "c3paper", "a4paper", "b4paper", "c4paper", "a5paper", "b5paper",
 139 "c5paper", "a6paper", "b6paper", "c6paper", "executivepaper", "legalpaper",
 140 "letterpaper", "b0j", "b1j", "b2j", "b3j", "b4j", "b5j", "b6j", 0};
 141
 142 const char * const known_class_paper_sizes[] = { "a4paper", "a5paper",
 143 "executivepaper", "legalpaper", "letterpaper", 0};
 144
 145 const char * const known_paper_margins[] = { "lmargin", "tmargin", "rmargin",
 146 "bmargin", "headheight", "headsep", "footskip", "columnsep", 0};
 147
 148 const char * const known_coded_paper_margins[] = { "leftmargin", "topmargin",
 149 "rightmargin", "bottommargin", "headheight", "headsep", "footskip",
 150 "columnsep", 0};
 151
 152 /// commands that can start an \if...\else...\endif sequence
 153 const char * const known_if_commands[] = {"if", "ifarydshln", "ifbraket",
 154 "ifcancel", "ifcolortbl", "ifeurosym", "ifmarginnote", "ifmmode", "ifpdf",
 155 "ifsidecap", "ifupgreek", 0};
 156
 157 const char * const known_basic_colors[] = {"blue", "black", "cyan", "green",
 158 "magenta", "red", "white", "yellow", 0};
 159
 160 const char * const known_basic_color_codes[] = {"#0000ff", "#000000", "#00ffff", "#00ff00",
 161 "#ff00ff", "#ff0000", "#ffffff", "#ffff00", 0};
 162
 163 /// conditional commands with three arguments like \@ifundefined{}{}{}
 164 const char * const known_if_3arg_commands[] = {"@ifundefined", "IfFileExists",
 165 0};
 166
 167 /// packages that work only in xetex
 168 /// polyglossia is handled separately
 169 const char * const known_xetex_packages[] = {"arabxetex", "fixlatvian",
 170 "fontbook", "fontwrap", "mathspec", "philokalia", "unisugar",
 171 "xeCJK", "xecolor", "xecyr", "xeindex", "xepersian", "xunicode", 0};
 172
 173 /// packages that are automatically skipped if loaded by LyX
 174 const char * const known_lyx_packages[] = {"amsbsy", "amsmath", "amssymb",
 175 "amstext", "amsthm", "array", "babel", "booktabs", "calc", "CJK", "color", "float",
 176 "fontspec", "graphicx", "hhline", "ifthen", "longtable", "makeidx", "multirow",
 177 "nomencl", "pdfpages", "rotating", "rotfloat", "splitidx", "setspace",
 178 "subscript", "textcomp", "ulem", "url", "varioref", "verbatim", "wrapfig",
 179 "xunicode", 0};
 180
 181 // codes used to remove packages that are loaded automatically by LyX.
 182 // Syntax: package_beg_sep<name>package_mid_sep<package loading code>package_end_sep
 183 const char package_beg_sep = '\001';
 184 const char package_mid_sep = '\002';
 185 const char package_end_sep = '\003';
 186
 187
 188 // returns true if at least one of the options in what has been found
 189 bool handle_opt(vector<string> & opts, char const * const * what, string & target)
 190 {
 191         if (opts.empty())
 192                 return false;
 193
 194         bool found = false;
 195         // the last language option is the document language (for babel and LyX)
 196         // the last size option is the document font size
 197         vector<string>::iterator it;
 198         vector<string>::iterator position = opts.begin();
 199         for (; *what; ++what) {
 200                 it = find(opts.begin(), opts.end(), *what);
 201                 if (it != opts.end()) {
 202                         if (it >= position) {
 203                                 found = true;
 204                                 target = *what;
 205                                 position = it;
 206                         }
 207                 }
 208         }
 209         return found;
 210 }
 211
 212
 213 void delete_opt(vector<string> & opts, char const * const * what)
 214 {
 215         if (opts.empty())
 216                 return;
 217
 218         // remove found options from the list
 219         // do this after handle_opt to avoid potential memory leaks
 220         vector<string>::iterator it;
 221         for (; *what; ++what) {
 222                 it = find(opts.begin(), opts.end(), *what);
 223                 if (it != opts.end())
 224                         opts.erase(it);
 225         }
 226 }
 227
 228
 229 /*!
 230  * Split a package options string (keyval format) into a vector.
 231  * Example input:
 232  *   authorformat=smallcaps,
 233  *   commabeforerest,
 234  *   titleformat=colonsep,
 235  *   bibformat={tabular,ibidem,numbered}
 236  */
 237 vector<string> split_options(string const & input)
 238 {
 239         vector<string> options;
 240         string option;
 241         Parser p(input);
 242         while (p.good()) {
 243                 Token const & t = p.get_token();
 244                 if (t.asInput() == ",") {
 245                         options.push_back(trimSpaceAndEol(option));
 246                         option.erase();
 247                 } else if (t.asInput() == "=") {
 248                         option += '=';
 249                         p.skip_spaces(true);
 250                         if (p.next_token().asInput() == "{")
 251                                 option += '{' + p.getArg('{', '}') + '}';
 252                 } else if (t.cat() != catSpace)
 253                         option += t.asInput();
 254         }
 255
 256         if (!option.empty())
 257                 options.push_back(trimSpaceAndEol(option));
 258
 259         return options;
 260 }
 261
 262
 263 /*!
 264  * Retrieve a keyval option "name={value with=sign}" named \p name from
 265  * \p options and return the value.
 266  * The found option is also removed from \p options.
 267  */
 268 string process_keyval_opt(vector<string> & options, string name)
 269 {
 270         for (size_t i = 0; i < options.size(); ++i) {
 271                 vector<string> option;
 272                 split(options[i], option, '=');
 273                 if (option.size() < 2)
 274                         continue;
 275                 if (option[0] == name) {
 276                         options.erase(options.begin() + i);
 277                         option.erase(option.begin());
 278                         return join(option, "=");
 279                 }
 280         }
 281         return "";
 282 }
 283
 284 } // anonymous namespace
 285
 286
 287 /**
 288  * known polyglossia language names (including variants)
 289  */
 290 const char * const Preamble::polyglossia_languages[] = {
 291 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 292 "nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
 293 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 294 "irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
 295 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 296 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
 297 "brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
 298 "lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 299 "welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
 300 "marathi", "spanish",
 301 "american", "ancient", "australian", "british", "monotonic", "newzealand",
 302 "polytonic", 0};
 303
 304 /**
 305  * the same as polyglossia_languages with .lyx names
 306  * please keep this in sync with polyglossia_languages line by line!
 307  */
 308 const char * const Preamble::coded_polyglossia_languages[] = {
 309 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 310 "nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
 311 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 312 "irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
 313 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 314 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
 315 "brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
 316 "lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 317 "welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
 318 "marathi", "spanish",
 319 "american", "ancientgreek", "australian", "british", "greek", "newzealand",
 320 "polutonikogreek", 0};
 321
 322
 323 bool Preamble::indentParagraphs() const
 324 {
 325         return h_paragraph_separation == "indent";
 326 }
 327
 328
 329 bool Preamble::isPackageUsed(string const & package) const
 330 {
 331         return used_packages.find(package) != used_packages.end();
 332 }
 333
 334
 335 vector<string> Preamble::getPackageOptions(string const & package) const
 336 {
 337         map<string, vector<string> >::const_iterator it = used_packages.find(package);
 338         if (it != used_packages.end())
 339                 return it->second;
 340         return vector<string>();
 341 }
 342
 343
 344 void Preamble::registerAutomaticallyLoadedPackage(std::string const & package)
 345 {
 346         auto_packages.insert(package);
 347 }
 348
 349
 350 void Preamble::addModule(string const & module)
 351 {
 352         used_modules.push_back(module);
 353 }
 354
 355
 356 void Preamble::suppressDate(bool suppress)
 357 {
 358         if (suppress)
 359                 h_suppress_date = "true";
 360         else
 361                 h_suppress_date = "false";
 362 }
 363
 364
 365 void Preamble::registerAuthor(std::string const & name)
 366 {
 367         Author author(from_utf8(name), empty_docstring());
 368         author.setUsed(true);
 369         authors_.record(author);
 370         h_tracking_changes = "true";
 371         h_output_changes = "true";
 372 }
 373
 374
 375 Author const & Preamble::getAuthor(std::string const & name) const
 376 {
 377         Author author(from_utf8(name), empty_docstring());
 378         for (AuthorList::Authors::const_iterator it = authors_.begin();
 379              it != authors_.end(); ++it)
 380                 if (*it == author)
 381                         return *it;
 382         static Author const dummy;
 383         return dummy;
 384 }
 385
 386
 387 void Preamble::add_package(string const & name, vector<string> & options)
 388 {
 389         // every package inherits the global options
 390         if (used_packages.find(name) == used_packages.end())
 391                 used_packages[name] = split_options(h_options);
 392
 393         vector<string> & v = used_packages[name];
 394         v.insert(v.end(), options.begin(), options.end());
 395         if (name == "jurabib") {
 396                 // Don't output the order argument (see the cite command
 397                 // handling code in text.cpp).
 398                 vector<string>::iterator end =
 399                         remove(options.begin(), options.end(), "natbiborder");
 400                 end = remove(options.begin(), end, "jurabiborder");
 401                 options.erase(end, options.end());
 402         }
 403 }
 404
 405
 406 namespace {
 407
 408 // Given is a string like "scaled=0.9", return 0.9 * 100
 409 string const scale_as_percentage(string const & scale)
 410 {
 411         string::size_type pos = scale.find('=');
 412         if (pos != string::npos) {
 413                 string value = scale.substr(pos + 1);
 414                 if (isStrDbl(value))
 415                         return convert<string>(100 * convert<double>(value));
 416         }
 417         // If the input string didn't match our expectations.
 418         // return the default value "100"
 419         return "100";
 420 }
 421
 422
 423 string remove_braces(string const & value)
 424 {
 425         if (value.empty())
 426                 return value;
 427         if (value[0] == '{' && value[value.length()-1] == '}')
 428                 return value.substr(1, value.length()-2);
 429         return value;
 430 }
 431
 432 } // anonymous namespace
 433
 434
 435 Preamble::Preamble() : one_language(true), title_layout_found(false)
 436 {
 437         //h_backgroundcolor;
 438         //h_boxbgcolor;
 439         h_biblio_style            = "plain";
 440         h_bibtex_command          = "default";
 441         h_cite_engine             = "basic";
 442         h_cite_engine_type        = "numerical";
 443         h_color                   = "#008000";
 444         h_defskip                 = "medskip";
 445         //h_float_placement;
 446         //h_fontcolor;
 447         h_fontencoding            = "default";
 448         h_font_roman              = "default";
 449         h_font_sans               = "default";
 450         h_font_typewriter         = "default";
 451         h_font_math               = "auto";
 452         h_font_default_family     = "default";
 453         h_use_non_tex_fonts       = "false";
 454         h_font_sc                 = "false";
 455         h_font_osf                = "false";
 456         h_font_sf_scale           = "100";
 457         h_font_tt_scale           = "100";
 458         h_graphics                = "default";
 459         h_default_output_format   = "default";
 460         h_html_be_strict          = "false";
 461         h_html_css_as_file        = "0";
 462         h_html_math_output        = "0";
 463         h_index                   = "Index";
 464         h_index_command           = "default";
 465         h_inputencoding           = "auto";
 466         h_justification           = "true";
 467         h_language                = "english";
 468         h_language_package        = "none";
 469         //h_listings_params;
 470         h_maintain_unincluded_children = "false";
 471         //h_margins;
 472         //h_notefontcolor;
 473         //h_options;
 474         h_output_changes          = "false";
 475         h_output_sync             = "0";
 476         //h_output_sync_macro
 477         h_papercolumns            = "1";
 478         h_paperfontsize           = "default";
 479         h_paperorientation        = "portrait";
 480         h_paperpagestyle          = "default";
 481         //h_papersides;
 482         h_papersize               = "default";
 483         h_paragraph_indentation   = "default";
 484         h_paragraph_separation    = "indent";
 485         //h_pdf_title;
 486         //h_pdf_author;
 487         //h_pdf_subject;
 488         //h_pdf_keywords;
 489         h_pdf_bookmarks           = "1";
 490         h_pdf_bookmarksnumbered   = "0";
 491         h_pdf_bookmarksopen       = "0";
 492         h_pdf_bookmarksopenlevel  = "1";
 493         h_pdf_breaklinks          = "0";
 494         h_pdf_pdfborder           = "0";
 495         h_pdf_colorlinks          = "0";
 496         h_pdf_backref             = "section";
 497         h_pdf_pdfusetitle         = "1";
 498         //h_pdf_pagemode;
 499         //h_pdf_quoted_options;
 500         h_quotes_language         = "english";
 501         h_secnumdepth             = "3";
 502         h_shortcut                = "idx";
 503         h_spacing                 = "single";
 504         h_suppress_date           = "false";
 505         h_textclass               = "article";
 506         h_tocdepth                = "3";
 507         h_tracking_changes        = "false";
 508         h_use_bibtopic            = "false";
 509         h_use_indices             = "false";
 510         h_use_geometry            = "false";
 511         h_use_default_options     = "false";
 512         h_use_hyperref            = "false";
 513         h_use_refstyle            = "0";
 514         h_use_packages["amsmath"]    = "1";
 515         h_use_packages["amssymb"]    = "0";
 516         h_use_packages["esint"]      = "1";
 517         h_use_packages["mhchem"]     = "0";
 518         h_use_packages["mathdots"]   = "0";
 519         h_use_packages["mathtools"]  = "0";
 520         h_use_packages["stmaryrd"]   = "0";
 521         h_use_packages["undertilde"] = "0";
 522 }
 523
 524
 525 void Preamble::handle_hyperref(vector<string> & options)
 526 {
 527         // FIXME swallow inputencoding changes that might surround the
 528         //       hyperref setup if it was written by LyX
 529         h_use_hyperref = "true";
 530         // swallow "unicode=true", since LyX does always write that
 531         vector<string>::iterator it =
 532                 find(options.begin(), options.end(), "unicode=true");
 533         if (it != options.end())
 534                 options.erase(it);
 535         it = find(options.begin(), options.end(), "pdfusetitle");
 536         if (it != options.end()) {
 537                 h_pdf_pdfusetitle = "1";
 538                 options.erase(it);
 539         }
 540         string bookmarks = process_keyval_opt(options, "bookmarks");
 541         if (bookmarks == "true")
 542                 h_pdf_bookmarks = "1";
 543         else if (bookmarks == "false")
 544                 h_pdf_bookmarks = "0";
 545         if (h_pdf_bookmarks == "1") {
 546                 string bookmarksnumbered =
 547                         process_keyval_opt(options, "bookmarksnumbered");
 548                 if (bookmarksnumbered == "true")
 549                         h_pdf_bookmarksnumbered = "1";
 550                 else if (bookmarksnumbered == "false")
 551                         h_pdf_bookmarksnumbered = "0";
 552                 string bookmarksopen =
 553                         process_keyval_opt(options, "bookmarksopen");
 554                 if (bookmarksopen == "true")
 555                         h_pdf_bookmarksopen = "1";
 556                 else if (bookmarksopen == "false")
 557                         h_pdf_bookmarksopen = "0";
 558                 if (h_pdf_bookmarksopen == "1") {
 559                         string bookmarksopenlevel =
 560                                 process_keyval_opt(options, "bookmarksopenlevel");
 561                         if (!bookmarksopenlevel.empty())
 562                                 h_pdf_bookmarksopenlevel = bookmarksopenlevel;
 563                 }
 564         }
 565         string breaklinks = process_keyval_opt(options, "breaklinks");
 566         if (breaklinks == "true")
 567                 h_pdf_breaklinks = "1";
 568         else if (breaklinks == "false")
 569                 h_pdf_breaklinks = "0";
 570         string pdfborder = process_keyval_opt(options, "pdfborder");
 571         if (pdfborder == "{0 0 0}")
 572                 h_pdf_pdfborder = "1";
 573         else if (pdfborder == "{0 0 1}")
 574                 h_pdf_pdfborder = "0";
 575         string backref = process_keyval_opt(options, "backref");
 576         if (!backref.empty())
 577                 h_pdf_backref = backref;
 578         string colorlinks = process_keyval_opt(options, "colorlinks");
 579         if (colorlinks == "true")
 580                 h_pdf_colorlinks = "1";
 581         else if (colorlinks == "false")
 582                 h_pdf_colorlinks = "0";
 583         string pdfpagemode = process_keyval_opt(options, "pdfpagemode");
 584         if (!pdfpagemode.empty())
 585                 h_pdf_pagemode = pdfpagemode;
 586         string pdftitle = process_keyval_opt(options, "pdftitle");
 587         if (!pdftitle.empty()) {
 588                 h_pdf_title = remove_braces(pdftitle);
 589         }
 590         string pdfauthor = process_keyval_opt(options, "pdfauthor");
 591         if (!pdfauthor.empty()) {
 592                 h_pdf_author = remove_braces(pdfauthor);
 593         }
 594         string pdfsubject = process_keyval_opt(options, "pdfsubject");
 595         if (!pdfsubject.empty())
 596                 h_pdf_subject = remove_braces(pdfsubject);
 597         string pdfkeywords = process_keyval_opt(options, "pdfkeywords");
 598         if (!pdfkeywords.empty())
 599                 h_pdf_keywords = remove_braces(pdfkeywords);
 600         if (!options.empty()) {
 601                 if (!h_pdf_quoted_options.empty())
 602                         h_pdf_quoted_options += ',';
 603                 h_pdf_quoted_options += join(options, ",");
 604                 options.clear();
 605         }
 606 }
 607
 608
 609 void Preamble::handle_geometry(vector<string> & options)
 610 {
 611         h_use_geometry = "true";
 612         vector<string>::iterator it;
 613         // paper orientation
 614         if ((it = find(options.begin(), options.end(), "landscape")) != options.end()) {
 615                 h_paperorientation = "landscape";
 616                 options.erase(it);
 617         }
 618         // paper size
 619         // keyval version: "paper=letter"
 620         string paper = process_keyval_opt(options, "paper");
 621         if (!paper.empty())
 622                 h_papersize = paper + "paper";
 623         // alternative version: "letterpaper"
 624         handle_opt(options, known_paper_sizes, h_papersize);
 625         delete_opt(options, known_paper_sizes);
 626         // page margins
 627         char const * const * margin = known_paper_margins;
 628         for (; *margin; ++margin) {
 629                 string value = process_keyval_opt(options, *margin);
 630                 if (!value.empty()) {
 631                         int k = margin - known_paper_margins;
 632                         string name = known_coded_paper_margins[k];
 633                         h_margins += '\\' + name + ' ' + value + '\n';
 634                 }
 635         }
 636 }
 637
 638
 639 void Preamble::handle_package(Parser &p, string const & name,
 640                               string const & opts, bool in_lyx_preamble)
 641 {
 642         vector<string> options = split_options(opts);
 643         add_package(name, options);
 644         string scale;
 645         char const * const * where = 0;
 646
 647         if (is_known(name, known_xetex_packages)) {
 648                 xetex = true;
 649                 h_use_non_tex_fonts = "true";
 650                 registerAutomaticallyLoadedPackage("fontspec");
 651                 if (h_inputencoding == "auto")
 652                         p.setEncoding("utf8");
 653         }
 654
 655         // roman fonts
 656         if (is_known(name, known_roman_fonts))
 657                 h_font_roman = name;
 658
 659         if (name == "fourier") {
 660                 h_font_roman = "utopia";
 661                 // when font uses real small capitals
 662                 if (opts == "expert")
 663                         h_font_sc = "true";
 664         }
 665
 666         else if (name == "mathpazo")
 667                 h_font_roman = "palatino";
 668
 669         else if (name == "mathptmx")
 670                 h_font_roman = "times";
 671
 672         // sansserif fonts
 673         if (is_known(name, known_sans_fonts)) {
 674                 h_font_sans = name;
 675                 if (!opts.empty()) {
 676                         scale = opts;
 677                         h_font_sf_scale = scale_as_percentage(scale);
 678                 }
 679         }
 680
 681         // typewriter fonts
 682         if (is_known(name, known_typewriter_fonts)) {
 683                 // fourier can be set as roman font _only_
 684                 // fourier as typewriter is handled in handling of \ttdefault
 685                 if (name != "fourier") {
 686                         h_font_typewriter = name;
 687                         if (!opts.empty()) {
 688                                 scale = opts;
 689                                 h_font_tt_scale = scale_as_percentage(scale);
 690                         }
 691                 }
 692         }
 693
 694         // font uses old-style figure
 695         if (name == "eco")
 696                 h_font_osf = "true";
 697
 698         // after the detection and handling of special cases, we can remove the
 699         // fonts, otherwise they would appear in the preamble, see bug #7856
 700         if (is_known(name, known_roman_fonts) || is_known(name, known_sans_fonts)
 701                 ||      is_known(name, known_typewriter_fonts))
 702                 ;
 703
 704         else if (name == "amsmath" || name == "amssymb" ||
 705                  name == "esint" || name == "mhchem" || name == "mathdots" ||
 706                  name == "mathtools" || name == "stmaryrd" ||
 707                  name == "undertilde")
 708                 h_use_packages[name] = "2";
 709
 710         else if (name == "babel") {
 711                 h_language_package = "default";
 712                 // One might think we would have to do nothing if babel is loaded
 713                 // without any options to prevent pollution of the preamble with this
 714                 // babel call in every roundtrip.
 715                 // But the user could have defined babel-specific things afterwards. So
 716                 // we need to keep it in the preamble to prevent cases like bug #7861.
 717                 if (!opts.empty()) {
 718                         // check if more than one option was used - used later for inputenc
 719                         if (options.begin() != options.end() - 1)
 720                                 one_language = false;
 721                         // babel takes the last language of the option of its \usepackage
 722                         // call as document language. If there is no such language option, the
 723                         // last language in the documentclass options is used.
 724                         handle_opt(options, known_languages, h_language);
 725                         // translate the babel name to a LyX name
 726                         h_language = babel2lyx(h_language);
 727                         // for Japanese we assume EUC-JP as encoding
 728                         // but we cannot determine the exact encoding and thus output also a note
 729                         if (h_language == "japanese") {
 730                                 h_inputencoding = "euc";
 731                                 p.setEncoding("EUC-JP");
 732                                 is_nonCJKJapanese = true;
 733                                 // in this case babel can be removed from the preamble
 734                                 registerAutomaticallyLoadedPackage("babel");
 735                         } else {
 736                                 // If babel is called with options, LyX puts them by default into the
 737                                 // document class options. This works for most languages, except
 738                                 // for Latvian, Lithuanian, Mongolian, Turkmen and Vietnamese and
 739                                 // perhaps in future others.
 740                                 // Therefore keep the babel call as it is as the user might have
 741                                 // reasons for it.
 742                                 h_preamble << "\\usepackage[" << opts << "]{babel}\n";
 743                         }
 744                         delete_opt(options, known_languages);
 745                 }
 746                 else
 747                         h_preamble << "\\usepackage{babel}\n";
 748         }
 749
 750         else if (name == "polyglossia") {
 751                 h_language_package = "default";
 752                 h_default_output_format = "pdf4";
 753                 h_use_non_tex_fonts = "true";
 754                 xetex = true;
 755                 registerAutomaticallyLoadedPackage("xunicode");
 756                 if (h_inputencoding == "auto")
 757                         p.setEncoding("utf8");
 758         }
 759
 760         else if (name == "CJK") {
 761                 // set the encoding to "auto" because it might be set to "default" by the babel handling
 762                 // and this would not be correct for CJK
 763                 if (h_inputencoding == "default")
 764                         h_inputencoding = "auto";
 765                 registerAutomaticallyLoadedPackage("CJK");
 766         }
 767
 768         else if (name == "fontenc") {
 769                 h_fontencoding = getStringFromVector(options, ",");
 770                 /* We could do the following for better round trip support,
 771                  * but this makes the document less portable, so I skip it:
 772                 if (h_fontencoding == lyxrc.fontenc)
 773                         h_fontencoding = "global";
 774                  */
 775                 options.clear();
 776         }
 777
 778         else if (name == "inputenc" || name == "luainputenc") {
 779                 // h_inputencoding is only set when there is not more than one
 780                 // inputenc option because otherwise h_inputencoding must be
 781                 // set to "auto" (the default encoding of the document language)
 782                 // Therefore check for the "," character.
 783                 // It is also only set when there is not more than one babel
 784                 // language option.
 785                 if (opts.find(",") == string::npos && one_language == true)
 786                         h_inputencoding = opts;
 787                 if (!options.empty())
 788                         p.setEncoding(options.back());
 789                 options.clear();
 790         }
 791
 792         else if (name == "srcltx") {
 793                 h_output_sync = "1";
 794                 if (!opts.empty()) {
 795                         h_output_sync_macro = "\\usepackage[" + opts + "]{srcltx}";
 796                         options.clear();
 797                 } else
 798                         h_output_sync_macro = "\\usepackage{srcltx}";
 799         }
 800
 801         else if (is_known(name, known_old_language_packages)) {
 802                 // known language packages from the times before babel
 803                 // if they are found and not also babel, they will be used as
 804                 // custom language package
 805                 h_language_package = "\\usepackage{" + name + "}";
 806         }
 807
 808         else if (name == "prettyref")
 809                 ; // ignore this FIXME: Use the package separator mechanism instead
 810
 811         else if (name == "lyxskak") {
 812                 // ignore this and its options
 813                 const char * const o[] = {"ps", "mover", 0};
 814                 delete_opt(options, o);
 815         }
 816
 817         else if (is_known(name, known_lyx_packages) && options.empty()) {
 818                 if (name == "splitidx")
 819                         h_use_indices = "true";
 820                 if (!in_lyx_preamble) {
 821                         h_preamble << package_beg_sep << name
 822                                    << package_mid_sep << "\\usepackage{"
 823                                    << name << '}';
 824                         if (p.next_token().cat() == catNewline ||
 825                             (p.next_token().cat() == catSpace &&
 826                              p.next_next_token().cat() == catNewline))
 827                                 h_preamble << '\n';
 828                         h_preamble << package_end_sep;
 829                 }
 830         }
 831
 832         else if (name == "geometry")
 833                 handle_geometry(options);
 834
 835         else if (name == "subfig")
 836                 ; // ignore this FIXME: Use the package separator mechanism instead
 837
 838         else if ((where = is_known(name, known_languages)))
 839                 h_language = known_coded_languages[where - known_languages];
 840
 841         else if (name == "natbib") {
 842                 h_biblio_style = "plainnat";
 843                 h_cite_engine = "natbib";
 844                 h_cite_engine_type = "authoryear";
 845                 vector<string>::iterator it =
 846                         find(options.begin(), options.end(), "authoryear");
 847                 if (it != options.end())
 848                         options.erase(it);
 849                 else {
 850                         it = find(options.begin(), options.end(), "numbers");
 851                         if (it != options.end()) {
 852                                 h_cite_engine_type = "numerical";
 853                                 options.erase(it);
 854                         }
 855                 }
 856         }
 857
 858         else if (name == "jurabib") {
 859                 h_biblio_style = "jurabib";
 860                 h_cite_engine = "jurabib";
 861                 h_cite_engine_type = "authoryear";
 862         }
 863
 864         else if (name == "hyperref")
 865                 handle_hyperref(options);
 866
 867         else if (!in_lyx_preamble) {
 868                 if (options.empty())
 869                         h_preamble << "\\usepackage{" << name << '}';
 870                 else {
 871                         h_preamble << "\\usepackage[" << opts << "]{"
 872                                    << name << '}';
 873                         options.clear();
 874                 }
 875                 if (p.next_token().cat() == catNewline ||
 876                     (p.next_token().cat() == catSpace &&
 877                      p.next_next_token().cat() == catNewline))
 878                         h_preamble << '\n';
 879         }
 880
 881         // We need to do something with the options...
 882         if (!options.empty())
 883                 cerr << "Ignoring options '" << join(options, ",")
 884                      << "' of package " << name << '.' << endl;
 885
 886         // remove the whitespace
 887         p.skip_spaces();
 888 }
 889
 890
 891 void Preamble::handle_if(Parser & p, bool in_lyx_preamble)
 892 {
 893         while (p.good()) {
 894                 Token t = p.get_token();
 895                 if (t.cat() == catEscape &&
 896                     is_known(t.cs(), known_if_commands))
 897                         handle_if(p, in_lyx_preamble);
 898                 else {
 899                         if (!in_lyx_preamble)
 900                                 h_preamble << t.asInput();
 901                         if (t.cat() == catEscape && t.cs() == "fi")
 902                                 return;
 903                 }
 904         }
 905 }
 906
 907
 908 bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
 909 {
 910         // set the quote language
 911         // LyX only knows the following quotes languages:
 912         // english, swedish, german, polish, french and danish
 913         // (quotes for "japanese" and "chinese-traditional" are missing because
 914         //  they wouldn't be useful: http://www.lyx.org/trac/ticket/6383)
 915         // conversion list taken from
 916         // http://en.wikipedia.org/wiki/Quotation_mark,_non-English_usage
 917         // (quotes for kazakh and interlingua are unknown)
 918         // danish
 919         if (is_known(h_language, known_danish_quotes_languages))
 920                 h_quotes_language = "danish";
 921         // french
 922         else if (is_known(h_language, known_french_quotes_languages))
 923                 h_quotes_language = "french";
 924         // german
 925         else if (is_known(h_language, known_german_quotes_languages))
 926                 h_quotes_language = "german";
 927         // polish
 928         else if (is_known(h_language, known_polish_quotes_languages))
 929                 h_quotes_language = "polish";
 930         // swedish
 931         else if (is_known(h_language, known_swedish_quotes_languages))
 932                 h_quotes_language = "swedish";
 933         //english
 934         else if (is_known(h_language, known_english_quotes_languages))
 935                 h_quotes_language = "english";
 936
 937         if (contains(h_float_placement, "H"))
 938                 registerAutomaticallyLoadedPackage("float");
 939         if (h_spacing != "single" && h_spacing != "default")
 940                 registerAutomaticallyLoadedPackage("setspace");
 941         if (h_use_packages["amsmath"] == "2") {
 942                 // amsbsy and amstext are already provided by amsmath
 943                 registerAutomaticallyLoadedPackage("amsbsy");
 944                 registerAutomaticallyLoadedPackage("amstext");
 945         }
 946
 947         // output the LyX file settings
 948         os << "#LyX file created by tex2lyx " << PACKAGE_VERSION << "\n"
 949            << "\\lyxformat " << LYX_FORMAT << '\n'
 950            << "\\begin_document\n"
 951            << "\\begin_header\n"
 952            << "\\textclass " << h_textclass << "\n";
 953         string const raw = subdoc ? empty_string() : h_preamble.str();
 954         if (!raw.empty()) {
 955                 os << "\\begin_preamble\n";
 956                 for (string::size_type i = 0; i < raw.size(); ++i) {
 957                         if (raw[i] == package_beg_sep) {
 958                                 // Here follows some package loading code that
 959                                 // must be skipped if the package is loaded
 960                                 // automatically.
 961                                 string::size_type j = raw.find(package_mid_sep, i);
 962                                 if (j == string::npos)
 963                                         return false;
 964                                 string::size_type k = raw.find(package_end_sep, j);
 965                                 if (k == string::npos)
 966                                         return false;
 967                                 string const package = raw.substr(i + 1, j - i - 1);
 968                                 string const replacement = raw.substr(j + 1, k - j - 1);
 969                                 if (auto_packages.find(package) == auto_packages.end())
 970                                         os << replacement;
 971                                 i = k;
 972                         } else
 973                                 os.put(raw[i]);
 974                 }
 975                 os << "\n\\end_preamble\n";
 976         }
 977         if (!h_options.empty())
 978                 os << "\\options " << h_options << "\n";
 979         os << "\\use_default_options " << h_use_default_options << "\n";
 980         if (!used_modules.empty()) {
 981                 os << "\\begin_modules\n";
 982                 vector<string>::const_iterator const end = used_modules.end();
 983                 vector<string>::const_iterator it = used_modules.begin();
 984                 for (; it != end; ++it)
 985                         os << *it << '\n';
 986                 os << "\\end_modules\n";
 987         }
 988         os << "\\maintain_unincluded_children " << h_maintain_unincluded_children << "\n"
 989            << "\\language " << h_language << "\n"
 990            << "\\language_package " << h_language_package << "\n"
 991            << "\\inputencoding " << h_inputencoding << "\n"
 992            << "\\fontencoding " << h_fontencoding << "\n"
 993            << "\\font_roman " << h_font_roman << "\n"
 994            << "\\font_sans " << h_font_sans << "\n"
 995            << "\\font_typewriter " << h_font_typewriter << "\n"
 996            << "\\font_math " << h_font_math << "\n"
 997            << "\\font_default_family " << h_font_default_family << "\n"
 998            << "\\use_non_tex_fonts " << h_use_non_tex_fonts << "\n"
 999            << "\\font_sc " << h_font_sc << "\n"
1000            << "\\font_osf " << h_font_osf << "\n"
1001            << "\\font_sf_scale " << h_font_sf_scale << "\n"
1002            << "\\font_tt_scale " << h_font_tt_scale << "\n"
1003            << "\\graphics " << h_graphics << "\n"
1004            << "\\default_output_format " << h_default_output_format << "\n"
1005            << "\\output_sync " << h_output_sync << "\n";
1006         if (h_output_sync == "1")
1007                 os << "\\output_sync_macro \"" << h_output_sync_macro << "\"\n";
1008         os << "\\bibtex_command " << h_bibtex_command << "\n"
1009            << "\\index_command " << h_index_command << "\n";
1010         if (!h_float_placement.empty())
1011                 os << "\\float_placement " << h_float_placement << "\n";
1012         os << "\\paperfontsize " << h_paperfontsize << "\n"
1013            << "\\spacing " << h_spacing << "\n"
1014            << "\\use_hyperref " << h_use_hyperref << '\n';
1015         if (h_use_hyperref == "true") {
1016                 if (!h_pdf_title.empty())
1017                         os << "\\pdf_title \"" << h_pdf_title << "\"\n";
1018                 if (!h_pdf_author.empty())
1019                         os << "\\pdf_author \"" << h_pdf_author << "\"\n";
1020                 if (!h_pdf_subject.empty())
1021                         os << "\\pdf_subject \"" << h_pdf_subject << "\"\n";
1022                 if (!h_pdf_keywords.empty())
1023                         os << "\\pdf_keywords \"" << h_pdf_keywords << "\"\n";
1024                 os << "\\pdf_bookmarks " << h_pdf_bookmarks << "\n"
1025                       "\\pdf_bookmarksnumbered " << h_pdf_bookmarksnumbered << "\n"
1026                       "\\pdf_bookmarksopen " << h_pdf_bookmarksopen << "\n"
1027                       "\\pdf_bookmarksopenlevel " << h_pdf_bookmarksopenlevel << "\n"
1028                       "\\pdf_breaklinks " << h_pdf_breaklinks << "\n"
1029                       "\\pdf_pdfborder " << h_pdf_pdfborder << "\n"
1030                       "\\pdf_colorlinks " << h_pdf_colorlinks << "\n"
1031                       "\\pdf_backref " << h_pdf_backref << "\n"
1032                       "\\pdf_pdfusetitle " << h_pdf_pdfusetitle << '\n';
1033                 if (!h_pdf_pagemode.empty())
1034                         os << "\\pdf_pagemode " << h_pdf_pagemode << '\n';
1035                 if (!h_pdf_quoted_options.empty())
1036                         os << "\\pdf_quoted_options \"" << h_pdf_quoted_options << "\"\n";
1037         }
1038         os << "\\papersize " << h_papersize << "\n"
1039            << "\\use_geometry " << h_use_geometry << '\n';
1040         for (map<string, string>::const_iterator it = h_use_packages.begin();
1041              it != h_use_packages.end(); ++it)
1042                 os << "\\use_package " << it->first << ' ' << it->second << '\n';
1043         os << "\\cite_engine " << h_cite_engine << '\n'
1044            << "\\cite_engine_type " << h_cite_engine_type << '\n'
1045            << "\\biblio_style " << h_biblio_style << "\n"
1046            << "\\use_bibtopic " << h_use_bibtopic << "\n"
1047            << "\\use_indices " << h_use_indices << "\n"
1048            << "\\paperorientation " << h_paperorientation << '\n'
1049            << "\\suppress_date " << h_suppress_date << '\n'
1050            << "\\justification " << h_justification << '\n'
1051            << "\\use_refstyle " << h_use_refstyle << '\n';
1052         if (!h_fontcolor.empty())
1053                 os << "\\fontcolor " << h_fontcolor << '\n';
1054         if (!h_notefontcolor.empty())
1055                 os << "\\notefontcolor " << h_notefontcolor << '\n';
1056         if (!h_backgroundcolor.empty())
1057                 os << "\\backgroundcolor " << h_backgroundcolor << '\n';
1058         if (!h_boxbgcolor.empty())
1059                 os << "\\boxbgcolor " << h_boxbgcolor << '\n';
1060         os << "\\index " << h_index << '\n'
1061            << "\\shortcut " << h_shortcut << '\n'
1062            << "\\color " << h_color << '\n'
1063            << "\\end_index\n";
1064         os << h_margins
1065            << "\\secnumdepth " << h_secnumdepth << "\n"
1066            << "\\tocdepth " << h_tocdepth << "\n"
1067            << "\\paragraph_separation " << h_paragraph_separation << "\n";
1068         if (h_paragraph_separation == "skip")
1069                 os << "\\defskip " << h_defskip << "\n";
1070         else
1071                 os << "\\paragraph_indentation " << h_paragraph_indentation << "\n";
1072         os << "\\quotes_language " << h_quotes_language << "\n"
1073            << "\\papercolumns " << h_papercolumns << "\n"
1074            << "\\papersides " << h_papersides << "\n"
1075            << "\\paperpagestyle " << h_paperpagestyle << "\n";
1076         if (!h_listings_params.empty())
1077                 os << "\\listings_params " << h_listings_params << "\n";
1078         os << "\\tracking_changes " << h_tracking_changes << "\n"
1079            << "\\output_changes " << h_output_changes << "\n"
1080            << "\\html_math_output " << h_html_math_output << "\n"
1081            << "\\html_css_as_file " << h_html_css_as_file << "\n"
1082            << "\\html_be_strict " << h_html_be_strict << "\n"
1083            << authors_
1084            << "\\end_header\n\n"
1085            << "\\begin_body\n";
1086         return true;
1087 }
1088
1089
1090 void Preamble::parse(Parser & p, string const & forceclass,
1091                      TeX2LyXDocClass & tc)
1092 {
1093         // initialize fixed types
1094         special_columns['D'] = 3;
1095         bool is_full_document = false;
1096         bool is_lyx_file = false;
1097         bool in_lyx_preamble = false;
1098
1099         // determine whether this is a full document or a fragment for inclusion
1100         while (p.good()) {
1101                 Token const & t = p.get_token();
1102
1103                 if (t.cat() == catEscape && t.cs() == "documentclass") {
1104                         is_full_document = true;
1105                         break;
1106                 }
1107         }
1108         p.reset();
1109
1110         while (is_full_document && p.good()) {
1111                 Token const & t = p.get_token();
1112
1113 #ifdef FILEDEBUG
1114                 cerr << "t: " << t << "\n";
1115 #endif
1116
1117                 //
1118                 // cat codes
1119                 //
1120                 if (!in_lyx_preamble &&
1121                     (t.cat() == catLetter ||
1122                      t.cat() == catSuper ||
1123                      t.cat() == catSub ||
1124                      t.cat() == catOther ||
1125                      t.cat() == catMath ||
1126                      t.cat() == catActive ||
1127                      t.cat() == catBegin ||
1128                      t.cat() == catEnd ||
1129                      t.cat() == catAlign ||
1130                      t.cat() == catParameter))
1131                         h_preamble << t.cs();
1132
1133                 else if (!in_lyx_preamble &&
1134                          (t.cat() == catSpace || t.cat() == catNewline))
1135                         h_preamble << t.asInput();
1136
1137                 else if (t.cat() == catComment) {
1138                         static regex const islyxfile("%% LyX .* created this file");
1139                         static regex const usercommands("User specified LaTeX commands");
1140
1141                         string const comment = t.asInput();
1142
1143                         // magically switch encoding default if it looks like XeLaTeX
1144                         static string const magicXeLaTeX =
1145                                 "% This document must be compiled with XeLaTeX ";
1146                         if (comment.size() > magicXeLaTeX.size()
1147                                   && comment.substr(0, magicXeLaTeX.size()) == magicXeLaTeX
1148                                   && h_inputencoding == "auto") {
1149                                 cerr << "XeLaTeX comment found, switching to UTF8\n";
1150                                 h_inputencoding = "utf8";
1151                         }
1152                         smatch sub;
1153                         if (regex_search(comment, sub, islyxfile)) {
1154                                 is_lyx_file = true;
1155                                 in_lyx_preamble = true;
1156                         } else if (is_lyx_file
1157                                    && regex_search(comment, sub, usercommands))
1158                                 in_lyx_preamble = false;
1159                         else if (!in_lyx_preamble)
1160                                 h_preamble << t.asInput();
1161                 }
1162
1163                 else if (t.cs() == "pagestyle")
1164                         h_paperpagestyle = p.verbatim_item();
1165
1166                 else if (t.cs() == "setdefaultlanguage") {
1167                         xetex = true;
1168                         // We don't yet care about non-language variant options
1169                         // because LyX doesn't support this yet, see bug #8214
1170                         if (p.hasOpt()) {
1171                                 string langopts = p.getOpt();
1172                                 // check if the option contains a variant, if yes, extract it
1173                                 string::size_type pos_var = langopts.find("variant");
1174                                 string::size_type i = langopts.find(',', pos_var);
1175                                 string::size_type k = langopts.find('=', pos_var);
1176                                 if (pos_var != string::npos){
1177                                         string variant;
1178                                         if (i == string::npos)
1179                                                 variant = langopts.substr(k + 1, langopts.length() - k - 2);
1180                                         else
1181                                                 variant = langopts.substr(k + 1, i - k - 1);
1182                                         h_language = variant;
1183                                 }
1184                                 p.verbatim_item();
1185                         } else
1186                                 h_language = p.verbatim_item();
1187                         //finally translate the poyglossia name to a LyX name
1188                         h_language = polyglossia2lyx(h_language);
1189                 }
1190
1191                 else if (t.cs() == "setotherlanguage") {
1192                         // We don't yet care about the option because LyX doesn't
1193                         // support this yet, see bug #8214
1194                         p.hasOpt() ? p.getOpt() : string();
1195                         p.verbatim_item();
1196                 }
1197
1198                 else if (t.cs() == "setmainfont") {
1199                         // we don't care about the option
1200                         p.hasOpt() ? p.getOpt() : string();
1201                         h_font_roman = p.getArg('{', '}');
1202                 }
1203
1204                 else if (t.cs() == "setsansfont" || t.cs() == "setmonofont") {
1205                         // LyX currently only supports the scale option
1206                         string scale;
1207                         if (p.hasOpt()) {
1208                                 string fontopts = p.getArg('[', ']');
1209                                 // check if the option contains a scaling, if yes, extract it
1210                                 string::size_type pos = fontopts.find("Scale");
1211                                 if (pos != string::npos) {
1212                                         string::size_type i = fontopts.find(',', pos);
1213                                         if (i == string::npos)
1214                                                 scale = scale_as_percentage(fontopts.substr(pos + 1));
1215                                         else
1216                                                 scale = scale_as_percentage(fontopts.substr(pos, i - pos));
1217                                 }
1218                         }
1219                         if (t.cs() == "setsansfont") {
1220                                 if (!scale.empty())
1221                                         h_font_sf_scale = scale;
1222                                 h_font_sans = p.getArg('{', '}');
1223                         } else {
1224                                 if (!scale.empty())
1225                                         h_font_tt_scale = scale;
1226                                 h_font_typewriter = p.getArg('{', '}');
1227                         }
1228                 }
1229
1230                 else if (t.cs() == "date") {
1231                         string argument = p.getArg('{', '}');
1232                         if (argument.empty())
1233                                 h_suppress_date = "true";
1234                         else
1235                                 h_preamble << t.asInput() << '{' << argument << '}';
1236                 }
1237
1238                 else if (t.cs() == "color") {
1239                         string const space =
1240                                 (p.hasOpt() ? p.getOpt() : string());
1241                         string argument = p.getArg('{', '}');
1242                         // check the case that a standard color is used
1243                         if (space.empty() && is_known(argument, known_basic_colors)) {
1244                                 h_fontcolor = rgbcolor2code(argument);
1245                                 preamble.registerAutomaticallyLoadedPackage("color");
1246                         } else if (space.empty() && argument == "document_fontcolor")
1247                                 preamble.registerAutomaticallyLoadedPackage("color");
1248                         // check the case that LyX's document_fontcolor is defined
1249                         // but not used for \color
1250                         else {
1251                                 h_preamble << t.asInput();
1252                                 if (!space.empty())
1253                                         h_preamble << space;
1254                                 h_preamble << '{' << argument << '}';
1255                                 // the color might already be set because \definecolor
1256                                 // is parsed before this
1257                                 h_fontcolor = "";
1258                         }
1259                 }
1260
1261                 else if (t.cs() == "pagecolor") {
1262                         string argument = p.getArg('{', '}');
1263                         // check the case that a standard color is used
1264                         if (is_known(argument, known_basic_colors)) {
1265                                 h_backgroundcolor = rgbcolor2code(argument);
1266                         } else if (argument == "page_backgroundcolor")
1267                                 preamble.registerAutomaticallyLoadedPackage("color");
1268                         // check the case that LyX's page_backgroundcolor is defined
1269                         // but not used for \pagecolor
1270                         else {
1271                                 h_preamble << t.asInput() << '{' << argument << '}';
1272                                 // the color might already be set because \definecolor
1273                                 // is parsed before this
1274                                 h_backgroundcolor = "";
1275                         }
1276                 }
1277
1278                 else if (t.cs() == "makeatletter") {
1279                         // LyX takes care of this
1280                         p.setCatCode('@', catLetter);
1281                 }
1282
1283                 else if (t.cs() == "makeatother") {
1284                         // LyX takes care of this
1285                         p.setCatCode('@', catOther);
1286                 }
1287
1288                 else if (t.cs() == "newcommand" || t.cs() == "newcommandx"
1289                       || t.cs() == "renewcommand" || t.cs() == "renewcommandx"
1290                       || t.cs() == "providecommand" || t.cs() == "providecommandx"
1291                                 || t.cs() == "DeclareRobustCommand"
1292                       || t.cs() == "DeclareRobustCommandx"
1293                                 || t.cs() == "ProvideTextCommandDefault"
1294                                 || t.cs() == "DeclareMathAccent") {
1295                         bool star = false;
1296                         if (p.next_token().character() == '*') {
1297                                 p.get_token();
1298                                 star = true;
1299                         }
1300                         string const name = p.verbatim_item();
1301                         string const opt1 = p.getFullOpt();
1302                         string const opt2 = p.getFullOpt();
1303                         string const body = p.verbatim_item();
1304                         // font settings
1305                         if (name == "\\rmdefault")
1306                                 if (is_known(body, known_roman_fonts))
1307                                         h_font_roman = body;
1308                         if (name == "\\sfdefault")
1309                                 if (is_known(body, known_sans_fonts))
1310                                         h_font_sans = body;
1311                         if (name == "\\ttdefault")
1312                                 if (is_known(body, known_typewriter_fonts))
1313                                         h_font_typewriter = body;
1314                         if (name == "\\familydefault") {
1315                                 string family = body;
1316                                 // remove leading "\"
1317                                 h_font_default_family = family.erase(0,1);
1318                         }
1319
1320                         // remove the lyxdot definition that is re-added by LyX
1321                         // if necessary
1322                         if (name == "\\lyxdot")
1323                                 in_lyx_preamble = true;
1324
1325                         // Add the command to the known commands
1326                         add_known_command(name, opt1, !opt2.empty(), from_utf8(body));
1327
1328                         // only non-lyxspecific stuff
1329                         if (!in_lyx_preamble) {
1330                                 ostringstream ss;
1331                                 ss << '\\' << t.cs();
1332                                 if (star)
1333                                         ss << '*';
1334                                 ss << '{' << name << '}' << opt1 << opt2
1335                                    << '{' << body << "}";
1336                                 h_preamble << ss.str();
1337 /*
1338                                 ostream & out = in_preamble ? h_preamble : os;
1339                                 out << "\\" << t.cs() << "{" << name << "}"
1340                                     << opts << "{" << body << "}";
1341 */
1342                         }
1343                 }
1344
1345                 else if (t.cs() == "documentclass") {
1346                         vector<string>::iterator it;
1347                         vector<string> opts = split_options(p.getArg('[', ']'));
1348                         handle_opt(opts, known_fontsizes, h_paperfontsize);
1349                         delete_opt(opts, known_fontsizes);
1350                         // delete "pt" at the end
1351                         string::size_type i = h_paperfontsize.find("pt");
1352                         if (i != string::npos)
1353                                 h_paperfontsize.erase(i);
1354                         // The documentclass options are always parsed before the options
1355                         // of the babel call so that a language cannot overwrite the babel
1356                         // options.
1357                         handle_opt(opts, known_languages, h_language);
1358                         delete_opt(opts, known_languages);
1359
1360                         // paper orientation
1361                         if ((it = find(opts.begin(), opts.end(), "landscape")) != opts.end()) {
1362                                 h_paperorientation = "landscape";
1363                                 opts.erase(it);
1364                         }
1365                         // paper sides
1366                         if ((it = find(opts.begin(), opts.end(), "oneside"))
1367                                  != opts.end()) {
1368                                 h_papersides = "1";
1369                                 opts.erase(it);
1370                         }
1371                         if ((it = find(opts.begin(), opts.end(), "twoside"))
1372                                  != opts.end()) {
1373                                 h_papersides = "2";
1374                                 opts.erase(it);
1375                         }
1376                         // paper columns
1377                         if ((it = find(opts.begin(), opts.end(), "onecolumn"))
1378                                  != opts.end()) {
1379                                 h_papercolumns = "1";
1380                                 opts.erase(it);
1381                         }
1382                         if ((it = find(opts.begin(), opts.end(), "twocolumn"))
1383                                  != opts.end()) {
1384                                 h_papercolumns = "2";
1385                                 opts.erase(it);
1386                         }
1387                         // paper sizes
1388                         // some size options are known to any document classes, other sizes
1389                         // are handled by the \geometry command of the geometry package
1390                         handle_opt(opts, known_class_paper_sizes, h_papersize);
1391                         delete_opt(opts, known_class_paper_sizes);
1392                         // the remaining options
1393                         h_options = join(opts, ",");
1394                         // FIXME This does not work for classes that have a
1395                         //       different name in LyX than in LaTeX
1396                         h_textclass = p.getArg('{', '}');
1397                 }
1398
1399                 else if (t.cs() == "usepackage") {
1400                         string const options = p.getArg('[', ']');
1401                         string const name = p.getArg('{', '}');
1402                         vector<string> vecnames;
1403                         split(name, vecnames, ',');
1404                         vector<string>::const_iterator it  = vecnames.begin();
1405                         vector<string>::const_iterator end = vecnames.end();
1406                         for (; it != end; ++it)
1407                                 handle_package(p, trimSpaceAndEol(*it), options,
1408                                                in_lyx_preamble);
1409                 }
1410
1411                 else if (t.cs() == "inputencoding") {
1412                         string const encoding = p.getArg('{','}');
1413                         h_inputencoding = encoding;
1414                         p.setEncoding(encoding);
1415                 }
1416
1417                 else if (t.cs() == "newenvironment") {
1418                         string const name = p.getArg('{', '}');
1419                         string const opt1 = p.getFullOpt();
1420                         string const opt2 = p.getFullOpt();
1421                         string const beg = p.verbatim_item();
1422                         string const end = p.verbatim_item();
1423                         if (!in_lyx_preamble) {
1424                                 h_preamble << "\\newenvironment{" << name
1425                                            << '}' << opt1 << opt2 << '{'
1426                                            << beg << "}{" << end << '}';
1427                         }
1428                         add_known_environment(name, opt1, !opt2.empty(),
1429                                               from_utf8(beg), from_utf8(end));
1430
1431                 }
1432
1433                 else if (t.cs() == "newtheorem") {
1434                         string const name = p.getArg('{', '}');
1435                         string const opt1 = p.getFullOpt();
1436                         string const opt2 = p.getFullOpt();
1437                         string const body = p.verbatim_item();
1438                         string const opt3 = p.getFullOpt();
1439
1440                         add_known_theorem(name, opt1, !opt2.empty(),
1441                                 from_utf8("\\newtheorem{" + name + '}' +
1442                                           opt1 + opt2 + '{' + body + '}' + opt3));
1443
1444                         if (!in_lyx_preamble)
1445                                 h_preamble << "\\newtheorem{" << name << '}'
1446                                            << opt1 << opt2 << '{' << '}' << opt3;
1447                 }
1448
1449                 else if (t.cs() == "def") {
1450                         string name = p.get_token().cs();
1451                         // In fact, name may be more than the name:
1452                         // In the test case of bug 8116
1453                         // name == "csname SF@gobble@opt \endcsname".
1454                         // Therefore, we need to use asInput() instead of cs().
1455                         while (p.next_token().cat() != catBegin)
1456                                 name += p.get_token().asInput();
1457                         if (!in_lyx_preamble)
1458                                 h_preamble << "\\def\\" << name << '{'
1459                                            << p.verbatim_item() << "}";
1460                 }
1461
1462                 else if (t.cs() == "newcolumntype") {
1463                         string const name = p.getArg('{', '}');
1464                         trimSpaceAndEol(name);
1465                         int nargs = 0;
1466                         string opts = p.getOpt();
1467                         if (!opts.empty()) {
1468                                 istringstream is(string(opts, 1));
1469                                 is >> nargs;
1470                         }
1471                         special_columns[name[0]] = nargs;
1472                         h_preamble << "\\newcolumntype{" << name << "}";
1473                         if (nargs)
1474                                 h_preamble << "[" << nargs << "]";
1475                         h_preamble << "{" << p.verbatim_item() << "}";
1476                 }
1477
1478                 else if (t.cs() == "setcounter") {
1479                         string const name = p.getArg('{', '}');
1480                         string const content = p.getArg('{', '}');
1481                         if (name == "secnumdepth")
1482                                 h_secnumdepth = content;
1483                         else if (name == "tocdepth")
1484                                 h_tocdepth = content;
1485                         else
1486                                 h_preamble << "\\setcounter{" << name << "}{" << content << "}";
1487                 }
1488
1489                 else if (t.cs() == "setlength") {
1490                         string const name = p.verbatim_item();
1491                         string const content = p.verbatim_item();
1492                         // the paragraphs are only not indented when \parindent is set to zero
1493                         if (name == "\\parindent" && content != "") {
1494                                 if (content[0] == '0')
1495                                         h_paragraph_separation = "skip";
1496                                 else
1497                                         h_paragraph_indentation = translate_len(content);
1498                         } else if (name == "\\parskip") {
1499                                 if (content == "\\smallskipamount")
1500                                         h_defskip = "smallskip";
1501                                 else if (content == "\\medskipamount")
1502                                         h_defskip = "medskip";
1503                                 else if (content == "\\bigskipamount")
1504                                         h_defskip = "bigskip";
1505                                 else
1506                                         h_defskip = content;
1507                         } else
1508                                 h_preamble << "\\setlength{" << name << "}{" << content << "}";
1509                 }
1510
1511                 else if (t.cs() == "onehalfspacing")
1512                         h_spacing = "onehalf";
1513
1514                 else if (t.cs() == "doublespacing")
1515                         h_spacing = "double";
1516
1517                 else if (t.cs() == "setstretch")
1518                         h_spacing = "other " + p.verbatim_item();
1519
1520                 else if (t.cs() == "synctex") {
1521                         // the scheme is \synctex=value
1522                         // where value can only be "1" or "-1"
1523                         h_output_sync = "1";
1524                         // there can be any character behind the value (e.g. a linebreak or a '\'
1525                         // therefore we extract it char by char
1526                         p.get_token();
1527                         string value = p.get_token().asInput();
1528                         if (value == "-")
1529                                 value += p.get_token().asInput();
1530                         h_output_sync_macro = "\\synctex=" + value;
1531                 }
1532
1533                 else if (t.cs() == "begin") {
1534                         string const name = p.getArg('{', '}');
1535                         if (name == "document")
1536                                 break;
1537                         h_preamble << "\\begin{" << name << "}";
1538                 }
1539
1540                 else if (t.cs() == "geometry") {
1541                         vector<string> opts = split_options(p.getArg('{', '}'));
1542                         handle_geometry(opts);
1543                 }
1544
1545                 else if (t.cs() == "definecolor") {
1546                         string const color = p.getArg('{', '}');
1547                         string const space = p.getArg('{', '}');
1548                         string const value = p.getArg('{', '}');
1549                         if (color == "document_fontcolor" && space == "rgb") {
1550                                 RGBColor c(RGBColorFromLaTeX(value));
1551                                 h_fontcolor = X11hexname(c);
1552                         } else if (color == "note_fontcolor" && space == "rgb") {
1553                                 RGBColor c(RGBColorFromLaTeX(value));
1554                                 h_notefontcolor = X11hexname(c);
1555                         } else if (color == "page_backgroundcolor" && space == "rgb") {
1556                                 RGBColor c(RGBColorFromLaTeX(value));
1557                                 h_backgroundcolor = X11hexname(c);
1558                         } else if (color == "shadecolor" && space == "rgb") {
1559                                 RGBColor c(RGBColorFromLaTeX(value));
1560                                 h_boxbgcolor = X11hexname(c);
1561                         } else {
1562                                 h_preamble << "\\definecolor{" << color
1563                                            << "}{" << space << "}{" << value
1564                                            << '}';
1565                         }
1566                 }
1567
1568                 else if (t.cs() == "bibliographystyle")
1569                         h_biblio_style = p.verbatim_item();
1570
1571                 else if (t.cs() == "jurabibsetup") {
1572                         // FIXME p.getArg('{', '}') is most probably wrong (it
1573                         //       does not handle nested braces).
1574                         //       Use p.verbatim_item() instead.
1575                         vector<string> jurabibsetup =
1576                                 split_options(p.getArg('{', '}'));
1577                         // add jurabibsetup to the jurabib package options
1578                         add_package("jurabib", jurabibsetup);
1579                         if (!jurabibsetup.empty()) {
1580                                 h_preamble << "\\jurabibsetup{"
1581                                            << join(jurabibsetup, ",") << '}';
1582                         }
1583                 }
1584
1585                 else if (t.cs() == "hypersetup") {
1586                         vector<string> hypersetup =
1587                                 split_options(p.verbatim_item());
1588                         // add hypersetup to the hyperref package options
1589                         handle_hyperref(hypersetup);
1590                         if (!hypersetup.empty()) {
1591                                 h_preamble << "\\hypersetup{"
1592                                            << join(hypersetup, ",") << '}';
1593                         }
1594                 }
1595
1596                 else if (is_known(t.cs(), known_if_3arg_commands)) {
1597                         // prevent misparsing of \usepackage if it is used
1598                         // as an argument (see e.g. our own output of
1599                         // \@ifundefined above)
1600                         string const arg1 = p.verbatim_item();
1601                         string const arg2 = p.verbatim_item();
1602                         string const arg3 = p.verbatim_item();
1603                         // test case \@ifundefined{date}{}{\date{}}
1604                         if (t.cs() == "@ifundefined" && arg1 == "date" &&
1605                             arg2.empty() && arg3 == "\\date{}") {
1606                                 h_suppress_date = "true";
1607                         // older tex2lyx versions did output
1608                         // \@ifundefined{definecolor}{\usepackage{color}}{}
1609                         } else if (t.cs() == "@ifundefined" &&
1610                                    arg1 == "definecolor" &&
1611                                    arg2 == "\\usepackage{color}" &&
1612                                    arg3.empty()) {
1613                                 if (!in_lyx_preamble)
1614                                         h_preamble << package_beg_sep
1615                                                    << "color"
1616                                                    << package_mid_sep
1617                                                    << "\\@ifundefined{definecolor}{color}{}"
1618                                                    << package_end_sep;
1619                         // test for case
1620                         //\@ifundefined{showcaptionsetup}{}{%
1621                         // \PassOptionsToPackage{caption=false}{subfig}}
1622                         // that LyX uses for subfloats
1623                         } else if (t.cs() == "@ifundefined" &&
1624                                    arg1 == "showcaptionsetup" && arg2.empty()
1625                                 && arg3 == "%\n \\PassOptionsToPackage{caption=false}{subfig}") {
1626                                 ; // do nothing
1627                         } else if (!in_lyx_preamble) {
1628                                 h_preamble << t.asInput()
1629                                            << '{' << arg1 << '}'
1630                                            << '{' << arg2 << '}'
1631                                            << '{' << arg3 << '}';
1632                         }
1633                 }
1634
1635                 else if (is_known(t.cs(), known_if_commands)) {
1636                         // must not parse anything in conditional code, since
1637                         // LyX would output the parsed contents unconditionally
1638                         if (!in_lyx_preamble)
1639                                 h_preamble << t.asInput();
1640                         handle_if(p, in_lyx_preamble);
1641                 }
1642
1643                 else if (!t.cs().empty() && !in_lyx_preamble)
1644                         h_preamble << '\\' << t.cs();
1645         }
1646
1647         // remove the whitespace
1648         p.skip_spaces();
1649
1650         // Force textclass if the user wanted it
1651         if (!forceclass.empty())
1652                 h_textclass = forceclass;
1653         tc.setName(h_textclass);
1654         if (!tc.load()) {
1655                 cerr << "Error: Could not read layout file for textclass \"" << h_textclass << "\"." << endl;
1656                 exit(EXIT_FAILURE);
1657         }
1658         if (h_papersides.empty()) {
1659                 ostringstream ss;
1660                 ss << tc.sides();
1661                 h_papersides = ss.str();
1662         }
1663
1664         // If the CJK package is used we cannot set the document language from
1665         // the babel options. Instead, we guess which language is used most
1666         // and set this one.
1667         default_language = h_language;
1668         if (is_full_document && auto_packages.find("CJK") != auto_packages.end()) {
1669                 p.pushPosition();
1670                 h_language = guessLanguage(p, default_language);
1671                 p.popPosition();
1672         }
1673 }
1674
1675
1676 string babel2lyx(string const & language)
1677 {
1678         char const * const * where = is_known(language, known_languages);
1679         if (where)
1680                 return known_coded_languages[where - known_languages];
1681         return language;
1682 }
1683
1684
1685 string Preamble::polyglossia2lyx(string const & language)
1686 {
1687         char const * const * where = is_known(language, polyglossia_languages);
1688         if (where)
1689                 return coded_polyglossia_languages[where - polyglossia_languages];
1690         return language;
1691 }
1692
1693
1694 string rgbcolor2code(string const & name)
1695 {
1696         char const * const * where = is_known(name, known_basic_colors);
1697         if (where) {
1698                 // "red", "green" etc
1699                 return known_basic_color_codes[where - known_basic_colors];
1700         }
1701         // "255,0,0", "0,255,0" etc
1702         RGBColor c(RGBColorFromLaTeX(name));
1703         return X11hexname(c);
1704 }
1705
1706 // }])
1707
1708
1709 } // namespace lyx