src/tex2lyx/Preamble.cpp

   1 /**
   2  * \file Preamble.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Uwe Stöhr
   8  *
   9  * Full author contact details are available in file CREDITS.
  10  */
  11
  12 // {[(
  13
  14 #include <config.h>
  15
  16 #include "Preamble.h"
  17 #include "tex2lyx.h"
  18
  19 #include "LayoutFile.h"
  20 #include "Layout.h"
  21 #include "Lexer.h"
  22 #include "TextClass.h"
  23
  24 #include "support/convert.h"
  25 #include "support/FileName.h"
  26 #include "support/filetools.h"
  27 #include "support/lstrings.h"
  28
  29 #include "support/regex.h"
  30
  31 #include <algorithm>
  32 #include <iostream>
  33
  34 using namespace std;
  35 using namespace lyx::support;
  36
  37
  38 namespace lyx {
  39
  40 // special columntypes
  41 extern map<char, int> special_columns;
  42
  43 Preamble preamble;
  44
  45 namespace {
  46
  47 // CJK languages are handled in text.cpp, polyglossia languages are listed
  48 // further down.
  49 /**
  50  * known babel language names (including synonyms)
  51  * not in standard babel: arabic, arabtex, armenian, belarusian, serbian-latin, thai
  52  * please keep this in sync with known_coded_languages line by line!
  53  */
  54 const char * const known_languages[] = {"acadian", "afrikaans", "albanian",
  55 "american", "arabic", "arabtex", "australian", "austrian", "bahasa", "bahasai",
  56 "bahasam", "basque", "belarusian", "brazil", "brazilian", "breton", "british",
  57 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
  58 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "francais",
  59 "french", "frenchb", "frenchle", "frenchpro", "galician", "german", "germanb",
  60 "greek", "hebrew", "hungarian", "icelandic", "indon", "indonesian", "interlingua",
  61 "irish", "italian", "japanese", "kazakh", "kurmanji", "latin", "latvian", "lithuanian",
  62 "lowersorbian", "lsorbian", "magyar", "malay", "meyalu", "mongolian", "naustrian",
  63 "newzealand", "ngerman", "ngermanb", "norsk", "nynorsk", "polutonikogreek", "polish",
  64 "portuges", "portuguese", "romanian", "russian", "russianb", "samin",
  65 "scottish", "serbian", "serbian-latin", "slovak", "slovene", "spanish",
  66 "swedish", "thai", "turkish", "turkmen", "ukraineb", "ukrainian",
  67 "uppersorbian", "UKenglish", "USenglish", "usorbian", "vietnam", "welsh",
  68 0};
  69
  70 /**
  71  * the same as known_languages with .lyx names
  72  * please keep this in sync with known_languages line by line!
  73  */
  74 const char * const known_coded_languages[] = {"french", "afrikaans", "albanian",
  75 "american", "arabic_arabi", "arabic_arabtex", "australian", "austrian", "bahasa", "bahasa",
  76 "bahasam", "basque", "belarusian", "brazilian", "brazilian", "breton", "british",
  77 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
  78 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "french",
  79 "french", "french", "french", "french", "galician", "german", "german",
  80 "greek", "hebrew", "magyar", "icelandic", "bahasa", "bahasa", "interlingua",
  81 "irish", "italian", "japanese", "kazakh", "kurmanji", "latin", "latvian", "lithuanian",
  82 "lowersorbian", "lowersorbian", "magyar", "bahasam", "bahasam", "mongolian", "naustrian",
  83 "newzealand", "ngerman", "ngerman", "norsk", "nynorsk", "polutonikogreek", "polish",
  84 "portuguese", "portuguese", "romanian", "russian", "russian", "samin",
  85 "scottish", "serbian", "serbian-latin", "slovak", "slovene", "spanish",
  86 "swedish", "thai", "turkish", "turkmen", "ukrainian", "ukrainian",
  87 "uppersorbian", "uppersorbian", "english", "english", "vietnamese", "welsh",
  88 0};
  89
  90 /// languages with danish quotes (.lyx names)
  91 const char * const known_danish_quotes_languages[] = {"danish", 0};
  92
  93 /// languages with english quotes (.lyx names)
  94 const char * const known_english_quotes_languages[] = {"american", "australian",
  95 "bahasa", "bahasam", "brazilian", "canadian", "chinese-simplified", "english",
  96 "esperanto", "hebrew", "irish", "korean", "newzealand", "portuguese", "scottish",
  97 "thai", 0};
  98
  99 /// languages with french quotes (.lyx names)
 100 const char * const known_french_quotes_languages[] = {"albanian",
 101 "arabic_arabi", "arabic_arabtex", "basque", "canadien", "catalan", "french",
 102 "galician", "greek", "italian", "norsk", "nynorsk", "polutonikogreek",
 103 "russian", "spanish", "spanish-mexico", "turkish", "turkmen", "ukrainian",
 104 "vietnamese", 0};
 105
 106 /// languages with german quotes (.lyx names)
 107 const char * const known_german_quotes_languages[] = {"austrian", "bulgarian",
 108 "czech", "german", "icelandic", "lithuanian", "lowersorbian", "naustrian",
 109 "ngerman", "serbian", "serbian-latin", "slovak", "slovene", "uppersorbian", 0};
 110
 111 /// languages with polish quotes (.lyx names)
 112 const char * const known_polish_quotes_languages[] = {"afrikaans", "croatian",
 113 "dutch", "estonian", "magyar", "polish", "romanian", 0};
 114
 115 /// languages with swedish quotes (.lyx names)
 116 const char * const known_swedish_quotes_languages[] = {"finnish",
 117 "swedish", 0};
 118
 119 /// known language packages from the times before babel
 120 const char * const known_old_language_packages[] = {"french", "frenchle",
 121 "frenchpro", "german", "ngerman", "pmfrench", 0};
 122
 123 char const * const known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
 124
 125 const char * const known_roman_fonts[] = { "ae", "beraserif", "bookman",
 126 "ccfonts", "chancery", "charter", "cmr", "fourier", "lmodern", "mathpazo",
 127 "mathptmx", "newcent", "utopia", 0};
 128
 129 const char * const known_sans_fonts[] = { "avant", "berasans", "cmbr", "cmss",
 130 "helvet", "lmss", 0};
 131
 132 const char * const known_typewriter_fonts[] = { "beramono", "cmtl", "cmtt",
 133 "courier", "lmtt", "luximono", "fourier", "lmodern", "mathpazo", "mathptmx",
 134 "newcent", 0};
 135
 136 const char * const known_paper_sizes[] = { "a0paper", "b0paper", "c0paper",
 137 "a1paper", "b1paper", "c1paper", "a2paper", "b2paper", "c2paper", "a3paper",
 138 "b3paper", "c3paper", "a4paper", "b4paper", "c4paper", "a5paper", "b5paper",
 139 "c5paper", "a6paper", "b6paper", "c6paper", "executivepaper", "legalpaper",
 140 "letterpaper", "b0j", "b1j", "b2j", "b3j", "b4j", "b5j", "b6j", 0};
 141
 142 const char * const known_class_paper_sizes[] = { "a4paper", "a5paper",
 143 "executivepaper", "legalpaper", "letterpaper", 0};
 144
 145 const char * const known_paper_margins[] = { "lmargin", "tmargin", "rmargin",
 146 "bmargin", "headheight", "headsep", "footskip", "columnsep", 0};
 147
 148 const char * const known_coded_paper_margins[] = { "leftmargin", "topmargin",
 149 "rightmargin", "bottommargin", "headheight", "headsep", "footskip",
 150 "columnsep", 0};
 151
 152 /// commands that can start an \if...\else...\endif sequence
 153 const char * const known_if_commands[] = {"if", "ifarydshln", "ifbraket",
 154 "ifcancel", "ifcolortbl", "ifeurosym", "ifmarginnote", "ifmmode", "ifpdf",
 155 "ifsidecap", "ifupgreek", 0};
 156
 157 const char * const known_basic_colors[] = {"blue", "black", "cyan", "green",
 158 "magenta", "red", "white", "yellow", 0};
 159
 160 const char * const known_basic_color_codes[] = {"#0000ff", "#000000", "#00ffff", "#00ff00",
 161 "#ff00ff", "#ff0000", "#ffffff", "#ffff00", 0};
 162
 163 /// conditional commands with three arguments like \@ifundefined{}{}{}
 164 const char * const known_if_3arg_commands[] = {"@ifundefined", "IfFileExists",
 165 0};
 166
 167 /// packages that work only in xetex
 168 /// polyglossia is handled separately
 169 const char * const known_xetex_packages[] = {"arabxetex", "fixlatvian",
 170 "fontbook", "fontwrap", "mathspec", "philokalia", "unisugar",
 171 "xeCJK", "xecolor", "xecyr", "xeindex", "xepersian", "xunicode", 0};
 172
 173 /// packages that are automatically skipped if loaded by LyX
 174 const char * const known_lyx_packages[] = {"amsbsy", "amsmath", "amssymb",
 175 "amstext", "amsthm", "array", "babel", "booktabs", "calc", "CJK", "color", "float",
 176 "fontspec", "graphicx", "hhline", "ifthen", "longtable", "makeidx", "multirow",
 177 "nomencl", "pdfpages", "rotating", "rotfloat", "splitidx", "setspace",
 178 "subscript", "textcomp", "ulem", "url", "varioref", "verbatim", "wrapfig",
 179 "xunicode", 0};
 180
 181 // codes used to remove packages that are loaded automatically by LyX.
 182 // Syntax: package_beg_sep<name>package_mid_sep<package loading code>package_end_sep
 183 const char package_beg_sep = '\001';
 184 const char package_mid_sep = '\002';
 185 const char package_end_sep = '\003';
 186
 187
 188 // returns true if at least one of the options in what has been found
 189 bool handle_opt(vector<string> & opts, char const * const * what, string & target)
 190 {
 191         if (opts.empty())
 192                 return false;
 193
 194         bool found = false;
 195         // the last language option is the document language (for babel and LyX)
 196         // the last size option is the document font size
 197         vector<string>::iterator it;
 198         vector<string>::iterator position = opts.begin();
 199         for (; *what; ++what) {
 200                 it = find(opts.begin(), opts.end(), *what);
 201                 if (it != opts.end()) {
 202                         if (it >= position) {
 203                                 found = true;
 204                                 target = *what;
 205                                 position = it;
 206                         }
 207                 }
 208         }
 209         return found;
 210 }
 211
 212
 213 void delete_opt(vector<string> & opts, char const * const * what)
 214 {
 215         if (opts.empty())
 216                 return;
 217
 218         // remove found options from the list
 219         // do this after handle_opt to avoid potential memory leaks
 220         vector<string>::iterator it;
 221         for (; *what; ++what) {
 222                 it = find(opts.begin(), opts.end(), *what);
 223                 if (it != opts.end())
 224                         opts.erase(it);
 225         }
 226 }
 227
 228
 229 /*!
 230  * Split a package options string (keyval format) into a vector.
 231  * Example input:
 232  *   authorformat=smallcaps,
 233  *   commabeforerest,
 234  *   titleformat=colonsep,
 235  *   bibformat={tabular,ibidem,numbered}
 236  */
 237 vector<string> split_options(string const & input)
 238 {
 239         vector<string> options;
 240         string option;
 241         Parser p(input);
 242         while (p.good()) {
 243                 Token const & t = p.get_token();
 244                 if (t.asInput() == ",") {
 245                         options.push_back(trimSpaceAndEol(option));
 246                         option.erase();
 247                 } else if (t.asInput() == "=") {
 248                         option += '=';
 249                         p.skip_spaces(true);
 250                         if (p.next_token().asInput() == "{")
 251                                 option += '{' + p.getArg('{', '}') + '}';
 252                 } else if (t.cat() != catSpace)
 253                         option += t.asInput();
 254         }
 255
 256         if (!option.empty())
 257                 options.push_back(trimSpaceAndEol(option));
 258
 259         return options;
 260 }
 261
 262
 263 /*!
 264  * Retrieve a keyval option "name={value with=sign}" named \p name from
 265  * \p options and return the value.
 266  * The found option is also removed from \p options.
 267  */
 268 string process_keyval_opt(vector<string> & options, string name)
 269 {
 270         for (size_t i = 0; i < options.size(); ++i) {
 271                 vector<string> option;
 272                 split(options[i], option, '=');
 273                 if (option.size() < 2)
 274                         continue;
 275                 if (option[0] == name) {
 276                         options.erase(options.begin() + i);
 277                         option.erase(option.begin());
 278                         return join(option, "=");
 279                 }
 280         }
 281         return "";
 282 }
 283
 284 } // anonymous namespace
 285
 286
 287 /**
 288  * known polyglossia language names (including variants)
 289  */
 290 const char * const Preamble::polyglossia_languages[] = {
 291 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 292 "nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
 293 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 294 "irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
 295 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 296 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
 297 "brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
 298 "lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 299 "welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
 300 "marathi", "spanish",
 301 "american", "ancient", "australian", "british", "monotonic", "newzealand",
 302 "polytonic", 0};
 303
 304 /**
 305  * the same as polyglossia_languages with .lyx names
 306  * please keep this in sync with polyglossia_languages line by line!
 307  */
 308 const char * const Preamble::coded_polyglossia_languages[] = {
 309 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 310 "nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
 311 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 312 "irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
 313 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 314 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
 315 "brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
 316 "lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 317 "welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
 318 "marathi", "spanish",
 319 "american", "ancientgreek", "australian", "british", "greek", "newzealand",
 320 "polutonikogreek", 0};
 321
 322
 323 bool Preamble::indentParagraphs() const
 324 {
 325         return h_paragraph_separation == "indent";
 326 }
 327
 328
 329 bool Preamble::isPackageUsed(string const & package) const
 330 {
 331         return used_packages.find(package) != used_packages.end();
 332 }
 333
 334
 335 vector<string> Preamble::getPackageOptions(string const & package) const
 336 {
 337         map<string, vector<string> >::const_iterator it = used_packages.find(package);
 338         if (it != used_packages.end())
 339                 return it->second;
 340         return vector<string>();
 341 }
 342
 343
 344 void Preamble::registerAutomaticallyLoadedPackage(std::string const & package)
 345 {
 346         auto_packages.insert(package);
 347 }
 348
 349
 350 void Preamble::addModule(string const & module)
 351 {
 352         used_modules.push_back(module);
 353 }
 354
 355
 356 void Preamble::suppressDate(bool suppress)
 357 {
 358         if (suppress)
 359                 h_suppress_date = "true";
 360         else
 361                 h_suppress_date = "false";
 362 }
 363
 364
 365 void Preamble::registerAuthor(std::string const & name)
 366 {
 367         Author author(from_utf8(name), empty_docstring());
 368         author.setUsed(true);
 369         authors_.record(author);
 370         h_tracking_changes = "true";
 371         h_output_changes = "true";
 372 }
 373
 374
 375 Author const & Preamble::getAuthor(std::string const & name) const
 376 {
 377         Author author(from_utf8(name), empty_docstring());
 378         for (AuthorList::Authors::const_iterator it = authors_.begin();
 379              it != authors_.end(); ++it)
 380                 if (*it == author)
 381                         return *it;
 382         static Author const dummy;
 383         return dummy;
 384 }
 385
 386
 387 void Preamble::add_package(string const & name, vector<string> & options)
 388 {
 389         // every package inherits the global options
 390         if (used_packages.find(name) == used_packages.end())
 391                 used_packages[name] = split_options(h_options);
 392
 393         vector<string> & v = used_packages[name];
 394         v.insert(v.end(), options.begin(), options.end());
 395         if (name == "jurabib") {
 396                 // Don't output the order argument (see the cite command
 397                 // handling code in text.cpp).
 398                 vector<string>::iterator end =
 399                         remove(options.begin(), options.end(), "natbiborder");
 400                 end = remove(options.begin(), end, "jurabiborder");
 401                 options.erase(end, options.end());
 402         }
 403 }
 404
 405
 406 namespace {
 407
 408 // Given is a string like "scaled=0.9", return 0.9 * 100
 409 string const scale_as_percentage(string const & scale)
 410 {
 411         string::size_type pos = scale.find('=');
 412         if (pos != string::npos) {
 413                 string value = scale.substr(pos + 1);
 414                 if (isStrDbl(value))
 415                         return convert<string>(100 * convert<double>(value));
 416         }
 417         // If the input string didn't match our expectations.
 418         // return the default value "100"
 419         return "100";
 420 }
 421
 422
 423 string remove_braces(string const & value)
 424 {
 425         if (value.empty())
 426                 return value;
 427         if (value[0] == '{' && value[value.length()-1] == '}')
 428                 return value.substr(1, value.length()-2);
 429         return value;
 430 }
 431
 432 } // anonymous namespace
 433
 434
 435 Preamble::Preamble() : one_language(true), title_layout_found(false)
 436 {
 437         //h_backgroundcolor;
 438         //h_boxbgcolor;
 439         h_biblio_style            = "plain";
 440         h_bibtex_command          = "default";
 441         h_cite_engine             = "basic";
 442         h_cite_engine_type        = "numerical";
 443         h_color                   = "#008000";
 444         h_defskip                 = "medskip";
 445         //h_float_placement;
 446         //h_fontcolor;
 447         h_fontencoding            = "default";
 448         h_font_roman              = "default";
 449         h_font_sans               = "default";
 450         h_font_typewriter         = "default";
 451         h_font_math               = "auto";
 452         h_font_default_family     = "default";
 453         h_use_non_tex_fonts       = "false";
 454         h_font_sc                 = "false";
 455         h_font_osf                = "false";
 456         h_font_sf_scale           = "100";
 457         h_font_tt_scale           = "100";
 458         h_graphics                = "default";
 459         h_default_output_format   = "default";
 460         h_html_be_strict          = "false";
 461         h_html_css_as_file        = "0";
 462         h_html_math_output        = "0";
 463         h_index                   = "Index";
 464         h_index_command           = "default";
 465         h_inputencoding           = "auto";
 466         h_justification           = "true";
 467         h_language                = "english";
 468         h_language_package        = "none";
 469         //h_listings_params;
 470         h_maintain_unincluded_children = "false";
 471         //h_margins;
 472         //h_notefontcolor;
 473         //h_options;
 474         h_output_changes          = "false";
 475         h_output_sync             = "0";
 476         //h_output_sync_macro
 477         h_papercolumns            = "1";
 478         h_paperfontsize           = "default";
 479         h_paperorientation        = "portrait";
 480         h_paperpagestyle          = "default";
 481         //h_papersides;
 482         h_papersize               = "default";
 483         h_paragraph_indentation   = "default";
 484         h_paragraph_separation    = "indent";
 485         //h_pdf_title;
 486         //h_pdf_author;
 487         //h_pdf_subject;
 488         //h_pdf_keywords;
 489         h_pdf_bookmarks           = "1";
 490         h_pdf_bookmarksnumbered   = "0";
 491         h_pdf_bookmarksopen       = "0";
 492         h_pdf_bookmarksopenlevel  = "1";
 493         h_pdf_breaklinks          = "0";
 494         h_pdf_pdfborder           = "0";
 495         h_pdf_colorlinks          = "0";
 496         h_pdf_backref             = "section";
 497         h_pdf_pdfusetitle         = "1";
 498         //h_pdf_pagemode;
 499         //h_pdf_quoted_options;
 500         h_quotes_language         = "english";
 501         h_secnumdepth             = "3";
 502         h_shortcut                = "idx";
 503         h_spacing                 = "single";
 504         h_suppress_date           = "false";
 505         h_textclass               = "article";
 506         h_tocdepth                = "3";
 507         h_tracking_changes        = "false";
 508         h_use_bibtopic            = "false";
 509         h_use_indices             = "false";
 510         h_use_geometry            = "false";
 511         h_use_default_options     = "false";
 512         h_use_hyperref            = "false";
 513         h_use_refstyle            = "0";
 514         h_use_packages["amsmath"]    = "1";
 515         h_use_packages["amssymb"]    = "0";
 516         h_use_packages["esint"]      = "1";
 517         h_use_packages["mhchem"]     = "0";
 518         h_use_packages["mathdots"]   = "0";
 519         h_use_packages["mathtools"]  = "0";
 520         h_use_packages["stackrel"]   = "0";
 521         h_use_packages["stmaryrd"]   = "0";
 522         h_use_packages["undertilde"] = "0";
 523 }
 524
 525
 526 void Preamble::handle_hyperref(vector<string> & options)
 527 {
 528         // FIXME swallow inputencoding changes that might surround the
 529         //       hyperref setup if it was written by LyX
 530         h_use_hyperref = "true";
 531         // swallow "unicode=true", since LyX does always write that
 532         vector<string>::iterator it =
 533                 find(options.begin(), options.end(), "unicode=true");
 534         if (it != options.end())
 535                 options.erase(it);
 536         it = find(options.begin(), options.end(), "pdfusetitle");
 537         if (it != options.end()) {
 538                 h_pdf_pdfusetitle = "1";
 539                 options.erase(it);
 540         }
 541         string bookmarks = process_keyval_opt(options, "bookmarks");
 542         if (bookmarks == "true")
 543                 h_pdf_bookmarks = "1";
 544         else if (bookmarks == "false")
 545                 h_pdf_bookmarks = "0";
 546         if (h_pdf_bookmarks == "1") {
 547                 string bookmarksnumbered =
 548                         process_keyval_opt(options, "bookmarksnumbered");
 549                 if (bookmarksnumbered == "true")
 550                         h_pdf_bookmarksnumbered = "1";
 551                 else if (bookmarksnumbered == "false")
 552                         h_pdf_bookmarksnumbered = "0";
 553                 string bookmarksopen =
 554                         process_keyval_opt(options, "bookmarksopen");
 555                 if (bookmarksopen == "true")
 556                         h_pdf_bookmarksopen = "1";
 557                 else if (bookmarksopen == "false")
 558                         h_pdf_bookmarksopen = "0";
 559                 if (h_pdf_bookmarksopen == "1") {
 560                         string bookmarksopenlevel =
 561                                 process_keyval_opt(options, "bookmarksopenlevel");
 562                         if (!bookmarksopenlevel.empty())
 563                                 h_pdf_bookmarksopenlevel = bookmarksopenlevel;
 564                 }
 565         }
 566         string breaklinks = process_keyval_opt(options, "breaklinks");
 567         if (breaklinks == "true")
 568                 h_pdf_breaklinks = "1";
 569         else if (breaklinks == "false")
 570                 h_pdf_breaklinks = "0";
 571         string pdfborder = process_keyval_opt(options, "pdfborder");
 572         if (pdfborder == "{0 0 0}")
 573                 h_pdf_pdfborder = "1";
 574         else if (pdfborder == "{0 0 1}")
 575                 h_pdf_pdfborder = "0";
 576         string backref = process_keyval_opt(options, "backref");
 577         if (!backref.empty())
 578                 h_pdf_backref = backref;
 579         string colorlinks = process_keyval_opt(options, "colorlinks");
 580         if (colorlinks == "true")
 581                 h_pdf_colorlinks = "1";
 582         else if (colorlinks == "false")
 583                 h_pdf_colorlinks = "0";
 584         string pdfpagemode = process_keyval_opt(options, "pdfpagemode");
 585         if (!pdfpagemode.empty())
 586                 h_pdf_pagemode = pdfpagemode;
 587         string pdftitle = process_keyval_opt(options, "pdftitle");
 588         if (!pdftitle.empty()) {
 589                 h_pdf_title = remove_braces(pdftitle);
 590         }
 591         string pdfauthor = process_keyval_opt(options, "pdfauthor");
 592         if (!pdfauthor.empty()) {
 593                 h_pdf_author = remove_braces(pdfauthor);
 594         }
 595         string pdfsubject = process_keyval_opt(options, "pdfsubject");
 596         if (!pdfsubject.empty())
 597                 h_pdf_subject = remove_braces(pdfsubject);
 598         string pdfkeywords = process_keyval_opt(options, "pdfkeywords");
 599         if (!pdfkeywords.empty())
 600                 h_pdf_keywords = remove_braces(pdfkeywords);
 601         if (!options.empty()) {
 602                 if (!h_pdf_quoted_options.empty())
 603                         h_pdf_quoted_options += ',';
 604                 h_pdf_quoted_options += join(options, ",");
 605                 options.clear();
 606         }
 607 }
 608
 609
 610 void Preamble::handle_geometry(vector<string> & options)
 611 {
 612         h_use_geometry = "true";
 613         vector<string>::iterator it;
 614         // paper orientation
 615         if ((it = find(options.begin(), options.end(), "landscape")) != options.end()) {
 616                 h_paperorientation = "landscape";
 617                 options.erase(it);
 618         }
 619         // paper size
 620         // keyval version: "paper=letter"
 621         string paper = process_keyval_opt(options, "paper");
 622         if (!paper.empty())
 623                 h_papersize = paper + "paper";
 624         // alternative version: "letterpaper"
 625         handle_opt(options, known_paper_sizes, h_papersize);
 626         delete_opt(options, known_paper_sizes);
 627         // page margins
 628         char const * const * margin = known_paper_margins;
 629         for (; *margin; ++margin) {
 630                 string value = process_keyval_opt(options, *margin);
 631                 if (!value.empty()) {
 632                         int k = margin - known_paper_margins;
 633                         string name = known_coded_paper_margins[k];
 634                         h_margins += '\\' + name + ' ' + value + '\n';
 635                 }
 636         }
 637 }
 638
 639
 640 void Preamble::handle_package(Parser &p, string const & name,
 641                               string const & opts, bool in_lyx_preamble)
 642 {
 643         vector<string> options = split_options(opts);
 644         add_package(name, options);
 645         string scale;
 646         char const * const * where = 0;
 647
 648         if (is_known(name, known_xetex_packages)) {
 649                 xetex = true;
 650                 h_use_non_tex_fonts = "true";
 651                 registerAutomaticallyLoadedPackage("fontspec");
 652                 if (h_inputencoding == "auto")
 653                         p.setEncoding("utf8");
 654         }
 655
 656         // roman fonts
 657         if (is_known(name, known_roman_fonts))
 658                 h_font_roman = name;
 659
 660         if (name == "fourier") {
 661                 h_font_roman = "utopia";
 662                 // when font uses real small capitals
 663                 if (opts == "expert")
 664                         h_font_sc = "true";
 665         }
 666
 667         else if (name == "mathpazo")
 668                 h_font_roman = "palatino";
 669
 670         else if (name == "mathptmx")
 671                 h_font_roman = "times";
 672
 673         // sansserif fonts
 674         if (is_known(name, known_sans_fonts)) {
 675                 h_font_sans = name;
 676                 if (!opts.empty()) {
 677                         scale = opts;
 678                         h_font_sf_scale = scale_as_percentage(scale);
 679                 }
 680         }
 681
 682         // typewriter fonts
 683         if (is_known(name, known_typewriter_fonts)) {
 684                 // fourier can be set as roman font _only_
 685                 // fourier as typewriter is handled in handling of \ttdefault
 686                 if (name != "fourier") {
 687                         h_font_typewriter = name;
 688                         if (!opts.empty()) {
 689                                 scale = opts;
 690                                 h_font_tt_scale = scale_as_percentage(scale);
 691                         }
 692                 }
 693         }
 694
 695         // font uses old-style figure
 696         if (name == "eco")
 697                 h_font_osf = "true";
 698
 699         // after the detection and handling of special cases, we can remove the
 700         // fonts, otherwise they would appear in the preamble, see bug #7856
 701         if (is_known(name, known_roman_fonts) || is_known(name, known_sans_fonts)
 702                 ||      is_known(name, known_typewriter_fonts))
 703                 ;
 704
 705         else if (name == "amsmath" || name == "amssymb" ||
 706                  name == "esint" || name == "mhchem" || name == "mathdots" ||
 707                  name == "mathtools" || name == "stackrel" ||
 708                  name == "stmaryrd" || name == "undertilde")
 709                 h_use_packages[name] = "2";
 710
 711         else if (name == "babel") {
 712                 h_language_package = "default";
 713                 // One might think we would have to do nothing if babel is loaded
 714                 // without any options to prevent pollution of the preamble with this
 715                 // babel call in every roundtrip.
 716                 // But the user could have defined babel-specific things afterwards. So
 717                 // we need to keep it in the preamble to prevent cases like bug #7861.
 718                 if (!opts.empty()) {
 719                         // check if more than one option was used - used later for inputenc
 720                         if (options.begin() != options.end() - 1)
 721                                 one_language = false;
 722                         // babel takes the last language of the option of its \usepackage
 723                         // call as document language. If there is no such language option, the
 724                         // last language in the documentclass options is used.
 725                         handle_opt(options, known_languages, h_language);
 726                         // translate the babel name to a LyX name
 727                         h_language = babel2lyx(h_language);
 728                         // for Japanese we assume EUC-JP as encoding
 729                         // but we cannot determine the exact encoding and thus output also a note
 730                         if (h_language == "japanese") {
 731                                 h_inputencoding = "euc";
 732                                 p.setEncoding("EUC-JP");
 733                                 is_nonCJKJapanese = true;
 734                                 // in this case babel can be removed from the preamble
 735                                 registerAutomaticallyLoadedPackage("babel");
 736                         } else {
 737                                 // If babel is called with options, LyX puts them by default into the
 738                                 // document class options. This works for most languages, except
 739                                 // for Latvian, Lithuanian, Mongolian, Turkmen and Vietnamese and
 740                                 // perhaps in future others.
 741                                 // Therefore keep the babel call as it is as the user might have
 742                                 // reasons for it.
 743                                 h_preamble << "\\usepackage[" << opts << "]{babel}\n";
 744                         }
 745                         delete_opt(options, known_languages);
 746                 }
 747                 else
 748                         h_preamble << "\\usepackage{babel}\n";
 749         }
 750
 751         else if (name == "polyglossia") {
 752                 h_language_package = "default";
 753                 h_default_output_format = "pdf4";
 754                 h_use_non_tex_fonts = "true";
 755                 xetex = true;
 756                 registerAutomaticallyLoadedPackage("xunicode");
 757                 if (h_inputencoding == "auto")
 758                         p.setEncoding("utf8");
 759         }
 760
 761         else if (name == "CJK") {
 762                 // set the encoding to "auto" because it might be set to "default" by the babel handling
 763                 // and this would not be correct for CJK
 764                 if (h_inputencoding == "default")
 765                         h_inputencoding = "auto";
 766                 registerAutomaticallyLoadedPackage("CJK");
 767         }
 768
 769         else if (name == "fontenc") {
 770                 h_fontencoding = getStringFromVector(options, ",");
 771                 /* We could do the following for better round trip support,
 772                  * but this makes the document less portable, so I skip it:
 773                 if (h_fontencoding == lyxrc.fontenc)
 774                         h_fontencoding = "global";
 775                  */
 776                 options.clear();
 777         }
 778
 779         else if (name == "inputenc" || name == "luainputenc") {
 780                 // h_inputencoding is only set when there is not more than one
 781                 // inputenc option because otherwise h_inputencoding must be
 782                 // set to "auto" (the default encoding of the document language)
 783                 // Therefore check for the "," character.
 784                 // It is also only set when there is not more than one babel
 785                 // language option.
 786                 if (opts.find(",") == string::npos && one_language == true)
 787                         h_inputencoding = opts;
 788                 if (!options.empty())
 789                         p.setEncoding(options.back());
 790                 options.clear();
 791         }
 792
 793         else if (name == "srcltx") {
 794                 h_output_sync = "1";
 795                 if (!opts.empty()) {
 796                         h_output_sync_macro = "\\usepackage[" + opts + "]{srcltx}";
 797                         options.clear();
 798                 } else
 799                         h_output_sync_macro = "\\usepackage{srcltx}";
 800         }
 801
 802         else if (is_known(name, known_old_language_packages)) {
 803                 // known language packages from the times before babel
 804                 // if they are found and not also babel, they will be used as
 805                 // custom language package
 806                 h_language_package = "\\usepackage{" + name + "}";
 807         }
 808
 809         else if (name == "prettyref")
 810                 ; // ignore this FIXME: Use the package separator mechanism instead
 811
 812         else if (name == "lyxskak") {
 813                 // ignore this and its options
 814                 const char * const o[] = {"ps", "mover", 0};
 815                 delete_opt(options, o);
 816         }
 817
 818         else if (is_known(name, known_lyx_packages) && options.empty()) {
 819                 if (name == "splitidx")
 820                         h_use_indices = "true";
 821                 if (!in_lyx_preamble) {
 822                         h_preamble << package_beg_sep << name
 823                                    << package_mid_sep << "\\usepackage{"
 824                                    << name << '}';
 825                         if (p.next_token().cat() == catNewline ||
 826                             (p.next_token().cat() == catSpace &&
 827                              p.next_next_token().cat() == catNewline))
 828                                 h_preamble << '\n';
 829                         h_preamble << package_end_sep;
 830                 }
 831         }
 832
 833         else if (name == "geometry")
 834                 handle_geometry(options);
 835
 836         else if (name == "subfig")
 837                 ; // ignore this FIXME: Use the package separator mechanism instead
 838
 839         else if ((where = is_known(name, known_languages)))
 840                 h_language = known_coded_languages[where - known_languages];
 841
 842         else if (name == "natbib") {
 843                 h_biblio_style = "plainnat";
 844                 h_cite_engine = "natbib";
 845                 h_cite_engine_type = "authoryear";
 846                 vector<string>::iterator it =
 847                         find(options.begin(), options.end(), "authoryear");
 848                 if (it != options.end())
 849                         options.erase(it);
 850                 else {
 851                         it = find(options.begin(), options.end(), "numbers");
 852                         if (it != options.end()) {
 853                                 h_cite_engine_type = "numerical";
 854                                 options.erase(it);
 855                         }
 856                 }
 857         }
 858
 859         else if (name == "jurabib") {
 860                 h_biblio_style = "jurabib";
 861                 h_cite_engine = "jurabib";
 862                 h_cite_engine_type = "authoryear";
 863         }
 864
 865         else if (name == "hyperref")
 866                 handle_hyperref(options);
 867
 868         else if (!in_lyx_preamble) {
 869                 if (options.empty())
 870                         h_preamble << "\\usepackage{" << name << '}';
 871                 else {
 872                         h_preamble << "\\usepackage[" << opts << "]{"
 873                                    << name << '}';
 874                         options.clear();
 875                 }
 876                 if (p.next_token().cat() == catNewline ||
 877                     (p.next_token().cat() == catSpace &&
 878                      p.next_next_token().cat() == catNewline))
 879                         h_preamble << '\n';
 880         }
 881
 882         // We need to do something with the options...
 883         if (!options.empty())
 884                 cerr << "Ignoring options '" << join(options, ",")
 885                      << "' of package " << name << '.' << endl;
 886
 887         // remove the whitespace
 888         p.skip_spaces();
 889 }
 890
 891
 892 void Preamble::handle_if(Parser & p, bool in_lyx_preamble)
 893 {
 894         while (p.good()) {
 895                 Token t = p.get_token();
 896                 if (t.cat() == catEscape &&
 897                     is_known(t.cs(), known_if_commands))
 898                         handle_if(p, in_lyx_preamble);
 899                 else {
 900                         if (!in_lyx_preamble)
 901                                 h_preamble << t.asInput();
 902                         if (t.cat() == catEscape && t.cs() == "fi")
 903                                 return;
 904                 }
 905         }
 906 }
 907
 908
 909 bool Preamble::writeLyXHeader(ostream & os, bool subdoc)
 910 {
 911         // set the quote language
 912         // LyX only knows the following quotes languages:
 913         // english, swedish, german, polish, french and danish
 914         // (quotes for "japanese" and "chinese-traditional" are missing because
 915         //  they wouldn't be useful: http://www.lyx.org/trac/ticket/6383)
 916         // conversion list taken from
 917         // http://en.wikipedia.org/wiki/Quotation_mark,_non-English_usage
 918         // (quotes for kazakh and interlingua are unknown)
 919         // danish
 920         if (is_known(h_language, known_danish_quotes_languages))
 921                 h_quotes_language = "danish";
 922         // french
 923         else if (is_known(h_language, known_french_quotes_languages))
 924                 h_quotes_language = "french";
 925         // german
 926         else if (is_known(h_language, known_german_quotes_languages))
 927                 h_quotes_language = "german";
 928         // polish
 929         else if (is_known(h_language, known_polish_quotes_languages))
 930                 h_quotes_language = "polish";
 931         // swedish
 932         else if (is_known(h_language, known_swedish_quotes_languages))
 933                 h_quotes_language = "swedish";
 934         //english
 935         else if (is_known(h_language, known_english_quotes_languages))
 936                 h_quotes_language = "english";
 937
 938         if (contains(h_float_placement, "H"))
 939                 registerAutomaticallyLoadedPackage("float");
 940         if (h_spacing != "single" && h_spacing != "default")
 941                 registerAutomaticallyLoadedPackage("setspace");
 942         if (h_use_packages["amsmath"] == "2") {
 943                 // amsbsy and amstext are already provided by amsmath
 944                 registerAutomaticallyLoadedPackage("amsbsy");
 945                 registerAutomaticallyLoadedPackage("amstext");
 946         }
 947
 948         // output the LyX file settings
 949         os << "#LyX file created by tex2lyx " << PACKAGE_VERSION << "\n"
 950            << "\\lyxformat " << LYX_FORMAT << '\n'
 951            << "\\begin_document\n"
 952            << "\\begin_header\n"
 953            << "\\textclass " << h_textclass << "\n";
 954         string const raw = subdoc ? empty_string() : h_preamble.str();
 955         if (!raw.empty()) {
 956                 os << "\\begin_preamble\n";
 957                 for (string::size_type i = 0; i < raw.size(); ++i) {
 958                         if (raw[i] == package_beg_sep) {
 959                                 // Here follows some package loading code that
 960                                 // must be skipped if the package is loaded
 961                                 // automatically.
 962                                 string::size_type j = raw.find(package_mid_sep, i);
 963                                 if (j == string::npos)
 964                                         return false;
 965                                 string::size_type k = raw.find(package_end_sep, j);
 966                                 if (k == string::npos)
 967                                         return false;
 968                                 string const package = raw.substr(i + 1, j - i - 1);
 969                                 string const replacement = raw.substr(j + 1, k - j - 1);
 970                                 if (auto_packages.find(package) == auto_packages.end())
 971                                         os << replacement;
 972                                 i = k;
 973                         } else
 974                                 os.put(raw[i]);
 975                 }
 976                 os << "\n\\end_preamble\n";
 977         }
 978         if (!h_options.empty())
 979                 os << "\\options " << h_options << "\n";
 980         os << "\\use_default_options " << h_use_default_options << "\n";
 981         if (!used_modules.empty()) {
 982                 os << "\\begin_modules\n";
 983                 vector<string>::const_iterator const end = used_modules.end();
 984                 vector<string>::const_iterator it = used_modules.begin();
 985                 for (; it != end; ++it)
 986                         os << *it << '\n';
 987                 os << "\\end_modules\n";
 988         }
 989         os << "\\maintain_unincluded_children " << h_maintain_unincluded_children << "\n"
 990            << "\\language " << h_language << "\n"
 991            << "\\language_package " << h_language_package << "\n"
 992            << "\\inputencoding " << h_inputencoding << "\n"
 993            << "\\fontencoding " << h_fontencoding << "\n"
 994            << "\\font_roman " << h_font_roman << "\n"
 995            << "\\font_sans " << h_font_sans << "\n"
 996            << "\\font_typewriter " << h_font_typewriter << "\n"
 997            << "\\font_math " << h_font_math << "\n"
 998            << "\\font_default_family " << h_font_default_family << "\n"
 999            << "\\use_non_tex_fonts " << h_use_non_tex_fonts << "\n"
1000            << "\\font_sc " << h_font_sc << "\n"
1001            << "\\font_osf " << h_font_osf << "\n"
1002            << "\\font_sf_scale " << h_font_sf_scale << "\n"
1003            << "\\font_tt_scale " << h_font_tt_scale << "\n"
1004            << "\\graphics " << h_graphics << "\n"
1005            << "\\default_output_format " << h_default_output_format << "\n"
1006            << "\\output_sync " << h_output_sync << "\n";
1007         if (h_output_sync == "1")
1008                 os << "\\output_sync_macro \"" << h_output_sync_macro << "\"\n";
1009         os << "\\bibtex_command " << h_bibtex_command << "\n"
1010            << "\\index_command " << h_index_command << "\n";
1011         if (!h_float_placement.empty())
1012                 os << "\\float_placement " << h_float_placement << "\n";
1013         os << "\\paperfontsize " << h_paperfontsize << "\n"
1014            << "\\spacing " << h_spacing << "\n"
1015            << "\\use_hyperref " << h_use_hyperref << '\n';
1016         if (h_use_hyperref == "true") {
1017                 if (!h_pdf_title.empty())
1018                         os << "\\pdf_title \"" << h_pdf_title << "\"\n";
1019                 if (!h_pdf_author.empty())
1020                         os << "\\pdf_author \"" << h_pdf_author << "\"\n";
1021                 if (!h_pdf_subject.empty())
1022                         os << "\\pdf_subject \"" << h_pdf_subject << "\"\n";
1023                 if (!h_pdf_keywords.empty())
1024                         os << "\\pdf_keywords \"" << h_pdf_keywords << "\"\n";
1025                 os << "\\pdf_bookmarks " << h_pdf_bookmarks << "\n"
1026                       "\\pdf_bookmarksnumbered " << h_pdf_bookmarksnumbered << "\n"
1027                       "\\pdf_bookmarksopen " << h_pdf_bookmarksopen << "\n"
1028                       "\\pdf_bookmarksopenlevel " << h_pdf_bookmarksopenlevel << "\n"
1029                       "\\pdf_breaklinks " << h_pdf_breaklinks << "\n"
1030                       "\\pdf_pdfborder " << h_pdf_pdfborder << "\n"
1031                       "\\pdf_colorlinks " << h_pdf_colorlinks << "\n"
1032                       "\\pdf_backref " << h_pdf_backref << "\n"
1033                       "\\pdf_pdfusetitle " << h_pdf_pdfusetitle << '\n';
1034                 if (!h_pdf_pagemode.empty())
1035                         os << "\\pdf_pagemode " << h_pdf_pagemode << '\n';
1036                 if (!h_pdf_quoted_options.empty())
1037                         os << "\\pdf_quoted_options \"" << h_pdf_quoted_options << "\"\n";
1038         }
1039         os << "\\papersize " << h_papersize << "\n"
1040            << "\\use_geometry " << h_use_geometry << '\n';
1041         for (map<string, string>::const_iterator it = h_use_packages.begin();
1042              it != h_use_packages.end(); ++it)
1043                 os << "\\use_package " << it->first << ' ' << it->second << '\n';
1044         os << "\\cite_engine " << h_cite_engine << '\n'
1045            << "\\cite_engine_type " << h_cite_engine_type << '\n'
1046            << "\\biblio_style " << h_biblio_style << "\n"
1047            << "\\use_bibtopic " << h_use_bibtopic << "\n"
1048            << "\\use_indices " << h_use_indices << "\n"
1049            << "\\paperorientation " << h_paperorientation << '\n'
1050            << "\\suppress_date " << h_suppress_date << '\n'
1051            << "\\justification " << h_justification << '\n'
1052            << "\\use_refstyle " << h_use_refstyle << '\n';
1053         if (!h_fontcolor.empty())
1054                 os << "\\fontcolor " << h_fontcolor << '\n';
1055         if (!h_notefontcolor.empty())
1056                 os << "\\notefontcolor " << h_notefontcolor << '\n';
1057         if (!h_backgroundcolor.empty())
1058                 os << "\\backgroundcolor " << h_backgroundcolor << '\n';
1059         if (!h_boxbgcolor.empty())
1060                 os << "\\boxbgcolor " << h_boxbgcolor << '\n';
1061         os << "\\index " << h_index << '\n'
1062            << "\\shortcut " << h_shortcut << '\n'
1063            << "\\color " << h_color << '\n'
1064            << "\\end_index\n";
1065         os << h_margins
1066            << "\\secnumdepth " << h_secnumdepth << "\n"
1067            << "\\tocdepth " << h_tocdepth << "\n"
1068            << "\\paragraph_separation " << h_paragraph_separation << "\n";
1069         if (h_paragraph_separation == "skip")
1070                 os << "\\defskip " << h_defskip << "\n";
1071         else
1072                 os << "\\paragraph_indentation " << h_paragraph_indentation << "\n";
1073         os << "\\quotes_language " << h_quotes_language << "\n"
1074            << "\\papercolumns " << h_papercolumns << "\n"
1075            << "\\papersides " << h_papersides << "\n"
1076            << "\\paperpagestyle " << h_paperpagestyle << "\n";
1077         if (!h_listings_params.empty())
1078                 os << "\\listings_params " << h_listings_params << "\n";
1079         os << "\\tracking_changes " << h_tracking_changes << "\n"
1080            << "\\output_changes " << h_output_changes << "\n"
1081            << "\\html_math_output " << h_html_math_output << "\n"
1082            << "\\html_css_as_file " << h_html_css_as_file << "\n"
1083            << "\\html_be_strict " << h_html_be_strict << "\n"
1084            << authors_
1085            << "\\end_header\n\n"
1086            << "\\begin_body\n";
1087         return true;
1088 }
1089
1090
1091 void Preamble::parse(Parser & p, string const & forceclass,
1092                      TeX2LyXDocClass & tc)
1093 {
1094         // initialize fixed types
1095         special_columns['D'] = 3;
1096         bool is_full_document = false;
1097         bool is_lyx_file = false;
1098         bool in_lyx_preamble = false;
1099
1100         // determine whether this is a full document or a fragment for inclusion
1101         while (p.good()) {
1102                 Token const & t = p.get_token();
1103
1104                 if (t.cat() == catEscape && t.cs() == "documentclass") {
1105                         is_full_document = true;
1106                         break;
1107                 }
1108         }
1109         p.reset();
1110
1111         while (is_full_document && p.good()) {
1112                 Token const & t = p.get_token();
1113
1114 #ifdef FILEDEBUG
1115                 cerr << "t: " << t << "\n";
1116 #endif
1117
1118                 //
1119                 // cat codes
1120                 //
1121                 if (!in_lyx_preamble &&
1122                     (t.cat() == catLetter ||
1123                      t.cat() == catSuper ||
1124                      t.cat() == catSub ||
1125                      t.cat() == catOther ||
1126                      t.cat() == catMath ||
1127                      t.cat() == catActive ||
1128                      t.cat() == catBegin ||
1129                      t.cat() == catEnd ||
1130                      t.cat() == catAlign ||
1131                      t.cat() == catParameter))
1132                         h_preamble << t.cs();
1133
1134                 else if (!in_lyx_preamble &&
1135                          (t.cat() == catSpace || t.cat() == catNewline))
1136                         h_preamble << t.asInput();
1137
1138                 else if (t.cat() == catComment) {
1139                         static regex const islyxfile("%% LyX .* created this file");
1140                         static regex const usercommands("User specified LaTeX commands");
1141
1142                         string const comment = t.asInput();
1143
1144                         // magically switch encoding default if it looks like XeLaTeX
1145                         static string const magicXeLaTeX =
1146                                 "% This document must be compiled with XeLaTeX ";
1147                         if (comment.size() > magicXeLaTeX.size()
1148                                   && comment.substr(0, magicXeLaTeX.size()) == magicXeLaTeX
1149                                   && h_inputencoding == "auto") {
1150                                 cerr << "XeLaTeX comment found, switching to UTF8\n";
1151                                 h_inputencoding = "utf8";
1152                         }
1153                         smatch sub;
1154                         if (regex_search(comment, sub, islyxfile)) {
1155                                 is_lyx_file = true;
1156                                 in_lyx_preamble = true;
1157                         } else if (is_lyx_file
1158                                    && regex_search(comment, sub, usercommands))
1159                                 in_lyx_preamble = false;
1160                         else if (!in_lyx_preamble)
1161                                 h_preamble << t.asInput();
1162                 }
1163
1164                 else if (t.cs() == "pagestyle")
1165                         h_paperpagestyle = p.verbatim_item();
1166
1167                 else if (t.cs() == "setdefaultlanguage") {
1168                         xetex = true;
1169                         // We don't yet care about non-language variant options
1170                         // because LyX doesn't support this yet, see bug #8214
1171                         if (p.hasOpt()) {
1172                                 string langopts = p.getOpt();
1173                                 // check if the option contains a variant, if yes, extract it
1174                                 string::size_type pos_var = langopts.find("variant");
1175                                 string::size_type i = langopts.find(',', pos_var);
1176                                 string::size_type k = langopts.find('=', pos_var);
1177                                 if (pos_var != string::npos){
1178                                         string variant;
1179                                         if (i == string::npos)
1180                                                 variant = langopts.substr(k + 1, langopts.length() - k - 2);
1181                                         else
1182                                                 variant = langopts.substr(k + 1, i - k - 1);
1183                                         h_language = variant;
1184                                 }
1185                                 p.verbatim_item();
1186                         } else
1187                                 h_language = p.verbatim_item();
1188                         //finally translate the poyglossia name to a LyX name
1189                         h_language = polyglossia2lyx(h_language);
1190                 }
1191
1192                 else if (t.cs() == "setotherlanguage") {
1193                         // We don't yet care about the option because LyX doesn't
1194                         // support this yet, see bug #8214
1195                         p.hasOpt() ? p.getOpt() : string();
1196                         p.verbatim_item();
1197                 }
1198
1199                 else if (t.cs() == "setmainfont") {
1200                         // we don't care about the option
1201                         p.hasOpt() ? p.getOpt() : string();
1202                         h_font_roman = p.getArg('{', '}');
1203                 }
1204
1205                 else if (t.cs() == "setsansfont" || t.cs() == "setmonofont") {
1206                         // LyX currently only supports the scale option
1207                         string scale;
1208                         if (p.hasOpt()) {
1209                                 string fontopts = p.getArg('[', ']');
1210                                 // check if the option contains a scaling, if yes, extract it
1211                                 string::size_type pos = fontopts.find("Scale");
1212                                 if (pos != string::npos) {
1213                                         string::size_type i = fontopts.find(',', pos);
1214                                         if (i == string::npos)
1215                                                 scale = scale_as_percentage(fontopts.substr(pos + 1));
1216                                         else
1217                                                 scale = scale_as_percentage(fontopts.substr(pos, i - pos));
1218                                 }
1219                         }
1220                         if (t.cs() == "setsansfont") {
1221                                 if (!scale.empty())
1222                                         h_font_sf_scale = scale;
1223                                 h_font_sans = p.getArg('{', '}');
1224                         } else {
1225                                 if (!scale.empty())
1226                                         h_font_tt_scale = scale;
1227                                 h_font_typewriter = p.getArg('{', '}');
1228                         }
1229                 }
1230
1231                 else if (t.cs() == "date") {
1232                         string argument = p.getArg('{', '}');
1233                         if (argument.empty())
1234                                 h_suppress_date = "true";
1235                         else
1236                                 h_preamble << t.asInput() << '{' << argument << '}';
1237                 }
1238
1239                 else if (t.cs() == "color") {
1240                         string const space =
1241                                 (p.hasOpt() ? p.getOpt() : string());
1242                         string argument = p.getArg('{', '}');
1243                         // check the case that a standard color is used
1244                         if (space.empty() && is_known(argument, known_basic_colors)) {
1245                                 h_fontcolor = rgbcolor2code(argument);
1246                                 preamble.registerAutomaticallyLoadedPackage("color");
1247                         } else if (space.empty() && argument == "document_fontcolor")
1248                                 preamble.registerAutomaticallyLoadedPackage("color");
1249                         // check the case that LyX's document_fontcolor is defined
1250                         // but not used for \color
1251                         else {
1252                                 h_preamble << t.asInput();
1253                                 if (!space.empty())
1254                                         h_preamble << space;
1255                                 h_preamble << '{' << argument << '}';
1256                                 // the color might already be set because \definecolor
1257                                 // is parsed before this
1258                                 h_fontcolor = "";
1259                         }
1260                 }
1261
1262                 else if (t.cs() == "pagecolor") {
1263                         string argument = p.getArg('{', '}');
1264                         // check the case that a standard color is used
1265                         if (is_known(argument, known_basic_colors)) {
1266                                 h_backgroundcolor = rgbcolor2code(argument);
1267                         } else if (argument == "page_backgroundcolor")
1268                                 preamble.registerAutomaticallyLoadedPackage("color");
1269                         // check the case that LyX's page_backgroundcolor is defined
1270                         // but not used for \pagecolor
1271                         else {
1272                                 h_preamble << t.asInput() << '{' << argument << '}';
1273                                 // the color might already be set because \definecolor
1274                                 // is parsed before this
1275                                 h_backgroundcolor = "";
1276                         }
1277                 }
1278
1279                 else if (t.cs() == "makeatletter") {
1280                         // LyX takes care of this
1281                         p.setCatCode('@', catLetter);
1282                 }
1283
1284                 else if (t.cs() == "makeatother") {
1285                         // LyX takes care of this
1286                         p.setCatCode('@', catOther);
1287                 }
1288
1289                 else if (t.cs() == "newcommand" || t.cs() == "newcommandx"
1290                       || t.cs() == "renewcommand" || t.cs() == "renewcommandx"
1291                       || t.cs() == "providecommand" || t.cs() == "providecommandx"
1292                                 || t.cs() == "DeclareRobustCommand"
1293                       || t.cs() == "DeclareRobustCommandx"
1294                                 || t.cs() == "ProvideTextCommandDefault"
1295                                 || t.cs() == "DeclareMathAccent") {
1296                         bool star = false;
1297                         if (p.next_token().character() == '*') {
1298                                 p.get_token();
1299                                 star = true;
1300                         }
1301                         string const name = p.verbatim_item();
1302                         string const opt1 = p.getFullOpt();
1303                         string const opt2 = p.getFullOpt();
1304                         string const body = p.verbatim_item();
1305                         // font settings
1306                         if (name == "\\rmdefault")
1307                                 if (is_known(body, known_roman_fonts))
1308                                         h_font_roman = body;
1309                         if (name == "\\sfdefault")
1310                                 if (is_known(body, known_sans_fonts))
1311                                         h_font_sans = body;
1312                         if (name == "\\ttdefault")
1313                                 if (is_known(body, known_typewriter_fonts))
1314                                         h_font_typewriter = body;
1315                         if (name == "\\familydefault") {
1316                                 string family = body;
1317                                 // remove leading "\"
1318                                 h_font_default_family = family.erase(0,1);
1319                         }
1320
1321                         // remove the lyxdot definition that is re-added by LyX
1322                         // if necessary
1323                         if (name == "\\lyxdot")
1324                                 in_lyx_preamble = true;
1325
1326                         // Add the command to the known commands
1327                         add_known_command(name, opt1, !opt2.empty(), from_utf8(body));
1328
1329                         // only non-lyxspecific stuff
1330                         if (!in_lyx_preamble) {
1331                                 ostringstream ss;
1332                                 ss << '\\' << t.cs();
1333                                 if (star)
1334                                         ss << '*';
1335                                 ss << '{' << name << '}' << opt1 << opt2
1336                                    << '{' << body << "}";
1337                                 h_preamble << ss.str();
1338 /*
1339                                 ostream & out = in_preamble ? h_preamble : os;
1340                                 out << "\\" << t.cs() << "{" << name << "}"
1341                                     << opts << "{" << body << "}";
1342 */
1343                         }
1344                 }
1345
1346                 else if (t.cs() == "documentclass") {
1347                         vector<string>::iterator it;
1348                         vector<string> opts = split_options(p.getArg('[', ']'));
1349                         handle_opt(opts, known_fontsizes, h_paperfontsize);
1350                         delete_opt(opts, known_fontsizes);
1351                         // delete "pt" at the end
1352                         string::size_type i = h_paperfontsize.find("pt");
1353                         if (i != string::npos)
1354                                 h_paperfontsize.erase(i);
1355                         // The documentclass options are always parsed before the options
1356                         // of the babel call so that a language cannot overwrite the babel
1357                         // options.
1358                         handle_opt(opts, known_languages, h_language);
1359                         delete_opt(opts, known_languages);
1360
1361                         // paper orientation
1362                         if ((it = find(opts.begin(), opts.end(), "landscape")) != opts.end()) {
1363                                 h_paperorientation = "landscape";
1364                                 opts.erase(it);
1365                         }
1366                         // paper sides
1367                         if ((it = find(opts.begin(), opts.end(), "oneside"))
1368                                  != opts.end()) {
1369                                 h_papersides = "1";
1370                                 opts.erase(it);
1371                         }
1372                         if ((it = find(opts.begin(), opts.end(), "twoside"))
1373                                  != opts.end()) {
1374                                 h_papersides = "2";
1375                                 opts.erase(it);
1376                         }
1377                         // paper columns
1378                         if ((it = find(opts.begin(), opts.end(), "onecolumn"))
1379                                  != opts.end()) {
1380                                 h_papercolumns = "1";
1381                                 opts.erase(it);
1382                         }
1383                         if ((it = find(opts.begin(), opts.end(), "twocolumn"))
1384                                  != opts.end()) {
1385                                 h_papercolumns = "2";
1386                                 opts.erase(it);
1387                         }
1388                         // paper sizes
1389                         // some size options are known to any document classes, other sizes
1390                         // are handled by the \geometry command of the geometry package
1391                         handle_opt(opts, known_class_paper_sizes, h_papersize);
1392                         delete_opt(opts, known_class_paper_sizes);
1393                         // the remaining options
1394                         h_options = join(opts, ",");
1395                         // FIXME This does not work for classes that have a
1396                         //       different name in LyX than in LaTeX
1397                         h_textclass = p.getArg('{', '}');
1398                 }
1399
1400                 else if (t.cs() == "usepackage") {
1401                         string const options = p.getArg('[', ']');
1402                         string const name = p.getArg('{', '}');
1403                         vector<string> vecnames;
1404                         split(name, vecnames, ',');
1405                         vector<string>::const_iterator it  = vecnames.begin();
1406                         vector<string>::const_iterator end = vecnames.end();
1407                         for (; it != end; ++it)
1408                                 handle_package(p, trimSpaceAndEol(*it), options,
1409                                                in_lyx_preamble);
1410                 }
1411
1412                 else if (t.cs() == "inputencoding") {
1413                         string const encoding = p.getArg('{','}');
1414                         h_inputencoding = encoding;
1415                         p.setEncoding(encoding);
1416                 }
1417
1418                 else if (t.cs() == "newenvironment") {
1419                         string const name = p.getArg('{', '}');
1420                         string const opt1 = p.getFullOpt();
1421                         string const opt2 = p.getFullOpt();
1422                         string const beg = p.verbatim_item();
1423                         string const end = p.verbatim_item();
1424                         if (!in_lyx_preamble) {
1425                                 h_preamble << "\\newenvironment{" << name
1426                                            << '}' << opt1 << opt2 << '{'
1427                                            << beg << "}{" << end << '}';
1428                         }
1429                         add_known_environment(name, opt1, !opt2.empty(),
1430                                               from_utf8(beg), from_utf8(end));
1431
1432                 }
1433
1434                 else if (t.cs() == "newtheorem") {
1435                         string const name = p.getArg('{', '}');
1436                         string const opt1 = p.getFullOpt();
1437                         string const opt2 = p.getFullOpt();
1438                         string const body = p.verbatim_item();
1439                         string const opt3 = p.getFullOpt();
1440
1441                         add_known_theorem(name, opt1, !opt2.empty(),
1442                                 from_utf8("\\newtheorem{" + name + '}' +
1443                                           opt1 + opt2 + '{' + body + '}' + opt3));
1444
1445                         if (!in_lyx_preamble)
1446                                 h_preamble << "\\newtheorem{" << name << '}'
1447                                            << opt1 << opt2 << '{' << '}' << opt3;
1448                 }
1449
1450                 else if (t.cs() == "def") {
1451                         string name = p.get_token().cs();
1452                         // In fact, name may be more than the name:
1453                         // In the test case of bug 8116
1454                         // name == "csname SF@gobble@opt \endcsname".
1455                         // Therefore, we need to use asInput() instead of cs().
1456                         while (p.next_token().cat() != catBegin)
1457                                 name += p.get_token().asInput();
1458                         if (!in_lyx_preamble)
1459                                 h_preamble << "\\def\\" << name << '{'
1460                                            << p.verbatim_item() << "}";
1461                 }
1462
1463                 else if (t.cs() == "newcolumntype") {
1464                         string const name = p.getArg('{', '}');
1465                         trimSpaceAndEol(name);
1466                         int nargs = 0;
1467                         string opts = p.getOpt();
1468                         if (!opts.empty()) {
1469                                 istringstream is(string(opts, 1));
1470                                 is >> nargs;
1471                         }
1472                         special_columns[name[0]] = nargs;
1473                         h_preamble << "\\newcolumntype{" << name << "}";
1474                         if (nargs)
1475                                 h_preamble << "[" << nargs << "]";
1476                         h_preamble << "{" << p.verbatim_item() << "}";
1477                 }
1478
1479                 else if (t.cs() == "setcounter") {
1480                         string const name = p.getArg('{', '}');
1481                         string const content = p.getArg('{', '}');
1482                         if (name == "secnumdepth")
1483                                 h_secnumdepth = content;
1484                         else if (name == "tocdepth")
1485                                 h_tocdepth = content;
1486                         else
1487                                 h_preamble << "\\setcounter{" << name << "}{" << content << "}";
1488                 }
1489
1490                 else if (t.cs() == "setlength") {
1491                         string const name = p.verbatim_item();
1492                         string const content = p.verbatim_item();
1493                         // the paragraphs are only not indented when \parindent is set to zero
1494                         if (name == "\\parindent" && content != "") {
1495                                 if (content[0] == '0')
1496                                         h_paragraph_separation = "skip";
1497                                 else
1498                                         h_paragraph_indentation = translate_len(content);
1499                         } else if (name == "\\parskip") {
1500                                 if (content == "\\smallskipamount")
1501                                         h_defskip = "smallskip";
1502                                 else if (content == "\\medskipamount")
1503                                         h_defskip = "medskip";
1504                                 else if (content == "\\bigskipamount")
1505                                         h_defskip = "bigskip";
1506                                 else
1507                                         h_defskip = content;
1508                         } else
1509                                 h_preamble << "\\setlength{" << name << "}{" << content << "}";
1510                 }
1511
1512                 else if (t.cs() == "onehalfspacing")
1513                         h_spacing = "onehalf";
1514
1515                 else if (t.cs() == "doublespacing")
1516                         h_spacing = "double";
1517
1518                 else if (t.cs() == "setstretch")
1519                         h_spacing = "other " + p.verbatim_item();
1520
1521                 else if (t.cs() == "synctex") {
1522                         // the scheme is \synctex=value
1523                         // where value can only be "1" or "-1"
1524                         h_output_sync = "1";
1525                         // there can be any character behind the value (e.g. a linebreak or a '\'
1526                         // therefore we extract it char by char
1527                         p.get_token();
1528                         string value = p.get_token().asInput();
1529                         if (value == "-")
1530                                 value += p.get_token().asInput();
1531                         h_output_sync_macro = "\\synctex=" + value;
1532                 }
1533
1534                 else if (t.cs() == "begin") {
1535                         string const name = p.getArg('{', '}');
1536                         if (name == "document")
1537                                 break;
1538                         h_preamble << "\\begin{" << name << "}";
1539                 }
1540
1541                 else if (t.cs() == "geometry") {
1542                         vector<string> opts = split_options(p.getArg('{', '}'));
1543                         handle_geometry(opts);
1544                 }
1545
1546                 else if (t.cs() == "definecolor") {
1547                         string const color = p.getArg('{', '}');
1548                         string const space = p.getArg('{', '}');
1549                         string const value = p.getArg('{', '}');
1550                         if (color == "document_fontcolor" && space == "rgb") {
1551                                 RGBColor c(RGBColorFromLaTeX(value));
1552                                 h_fontcolor = X11hexname(c);
1553                         } else if (color == "note_fontcolor" && space == "rgb") {
1554                                 RGBColor c(RGBColorFromLaTeX(value));
1555                                 h_notefontcolor = X11hexname(c);
1556                         } else if (color == "page_backgroundcolor" && space == "rgb") {
1557                                 RGBColor c(RGBColorFromLaTeX(value));
1558                                 h_backgroundcolor = X11hexname(c);
1559                         } else if (color == "shadecolor" && space == "rgb") {
1560                                 RGBColor c(RGBColorFromLaTeX(value));
1561                                 h_boxbgcolor = X11hexname(c);
1562                         } else {
1563                                 h_preamble << "\\definecolor{" << color
1564                                            << "}{" << space << "}{" << value
1565                                            << '}';
1566                         }
1567                 }
1568
1569                 else if (t.cs() == "bibliographystyle")
1570                         h_biblio_style = p.verbatim_item();
1571
1572                 else if (t.cs() == "jurabibsetup") {
1573                         // FIXME p.getArg('{', '}') is most probably wrong (it
1574                         //       does not handle nested braces).
1575                         //       Use p.verbatim_item() instead.
1576                         vector<string> jurabibsetup =
1577                                 split_options(p.getArg('{', '}'));
1578                         // add jurabibsetup to the jurabib package options
1579                         add_package("jurabib", jurabibsetup);
1580                         if (!jurabibsetup.empty()) {
1581                                 h_preamble << "\\jurabibsetup{"
1582                                            << join(jurabibsetup, ",") << '}';
1583                         }
1584                 }
1585
1586                 else if (t.cs() == "hypersetup") {
1587                         vector<string> hypersetup =
1588                                 split_options(p.verbatim_item());
1589                         // add hypersetup to the hyperref package options
1590                         handle_hyperref(hypersetup);
1591                         if (!hypersetup.empty()) {
1592                                 h_preamble << "\\hypersetup{"
1593                                            << join(hypersetup, ",") << '}';
1594                         }
1595                 }
1596
1597                 else if (is_known(t.cs(), known_if_3arg_commands)) {
1598                         // prevent misparsing of \usepackage if it is used
1599                         // as an argument (see e.g. our own output of
1600                         // \@ifundefined above)
1601                         string const arg1 = p.verbatim_item();
1602                         string const arg2 = p.verbatim_item();
1603                         string const arg3 = p.verbatim_item();
1604                         // test case \@ifundefined{date}{}{\date{}}
1605                         if (t.cs() == "@ifundefined" && arg1 == "date" &&
1606                             arg2.empty() && arg3 == "\\date{}") {
1607                                 h_suppress_date = "true";
1608                         // older tex2lyx versions did output
1609                         // \@ifundefined{definecolor}{\usepackage{color}}{}
1610                         } else if (t.cs() == "@ifundefined" &&
1611                                    arg1 == "definecolor" &&
1612                                    arg2 == "\\usepackage{color}" &&
1613                                    arg3.empty()) {
1614                                 if (!in_lyx_preamble)
1615                                         h_preamble << package_beg_sep
1616                                                    << "color"
1617                                                    << package_mid_sep
1618                                                    << "\\@ifundefined{definecolor}{color}{}"
1619                                                    << package_end_sep;
1620                         // test for case
1621                         //\@ifundefined{showcaptionsetup}{}{%
1622                         // \PassOptionsToPackage{caption=false}{subfig}}
1623                         // that LyX uses for subfloats
1624                         } else if (t.cs() == "@ifundefined" &&
1625                                    arg1 == "showcaptionsetup" && arg2.empty()
1626                                 && arg3 == "%\n \\PassOptionsToPackage{caption=false}{subfig}") {
1627                                 ; // do nothing
1628                         } else if (!in_lyx_preamble) {
1629                                 h_preamble << t.asInput()
1630                                            << '{' << arg1 << '}'
1631                                            << '{' << arg2 << '}'
1632                                            << '{' << arg3 << '}';
1633                         }
1634                 }
1635
1636                 else if (is_known(t.cs(), known_if_commands)) {
1637                         // must not parse anything in conditional code, since
1638                         // LyX would output the parsed contents unconditionally
1639                         if (!in_lyx_preamble)
1640                                 h_preamble << t.asInput();
1641                         handle_if(p, in_lyx_preamble);
1642                 }
1643
1644                 else if (!t.cs().empty() && !in_lyx_preamble)
1645                         h_preamble << '\\' << t.cs();
1646         }
1647
1648         // remove the whitespace
1649         p.skip_spaces();
1650
1651         // Force textclass if the user wanted it
1652         if (!forceclass.empty())
1653                 h_textclass = forceclass;
1654         tc.setName(h_textclass);
1655         if (!tc.load()) {
1656                 cerr << "Error: Could not read layout file for textclass \"" << h_textclass << "\"." << endl;
1657                 exit(EXIT_FAILURE);
1658         }
1659         if (h_papersides.empty()) {
1660                 ostringstream ss;
1661                 ss << tc.sides();
1662                 h_papersides = ss.str();
1663         }
1664
1665         // If the CJK package is used we cannot set the document language from
1666         // the babel options. Instead, we guess which language is used most
1667         // and set this one.
1668         default_language = h_language;
1669         if (is_full_document && auto_packages.find("CJK") != auto_packages.end()) {
1670                 p.pushPosition();
1671                 h_language = guessLanguage(p, default_language);
1672                 p.popPosition();
1673         }
1674 }
1675
1676
1677 string babel2lyx(string const & language)
1678 {
1679         char const * const * where = is_known(language, known_languages);
1680         if (where)
1681                 return known_coded_languages[where - known_languages];
1682         return language;
1683 }
1684
1685
1686 string Preamble::polyglossia2lyx(string const & language)
1687 {
1688         char const * const * where = is_known(language, polyglossia_languages);
1689         if (where)
1690                 return coded_polyglossia_languages[where - polyglossia_languages];
1691         return language;
1692 }
1693
1694
1695 string rgbcolor2code(string const & name)
1696 {
1697         char const * const * where = is_known(name, known_basic_colors);
1698         if (where) {
1699                 // "red", "green" etc
1700                 return known_basic_color_codes[where - known_basic_colors];
1701         }
1702         // "255,0,0", "0,255,0" etc
1703         RGBColor c(RGBColorFromLaTeX(name));
1704         return X11hexname(c);
1705 }
1706
1707 // }])
1708
1709
1710 } // namespace lyx