]> git.lyx.org Git - lyx.git/blob - src/tex2lyx/Preamble.cpp
tex2lyx: update quote handling
[lyx.git] / src / tex2lyx / Preamble.cpp
1 /**
2  * \file Preamble.cpp
3  * This file is part of LyX, the document processor.
4  * Licence details can be found in the file COPYING.
5  *
6  * \author André Pönitz
7  * \author Uwe Stöhr
8  *
9  * Full author contact details are available in file CREDITS.
10  */
11
12 // {[(
13
14 #include <config.h>
15
16 #include "Preamble.h"
17 #include "tex2lyx.h"
18
19 #include "Encoding.h"
20 #include "LayoutFile.h"
21 #include "Layout.h"
22 #include "Lexer.h"
23 #include "TextClass.h"
24 #include "version.h"
25
26 #include "support/convert.h"
27 #include "support/FileName.h"
28 #include "support/filetools.h"
29 #include "support/lstrings.h"
30
31 #include "support/regex.h"
32
33 #include <algorithm>
34 #include <iostream>
35
36 using namespace std;
37 using namespace lyx::support;
38
39
40 namespace lyx {
41
42 Preamble preamble;
43
44 namespace {
45
46 // CJK languages are handled in text.cpp, polyglossia languages are listed
47 // further down.
48 /**
49  * known babel language names (including synonyms)
50  * not in standard babel: arabic, arabtex, armenian, belarusian, serbian-latin, thai
51  * please keep this in sync with known_coded_languages line by line!
52  */
53 const char * const known_languages[] = {"acadian", "afrikaans", "albanian",
54 "american", "arabic", "arabtex", "australian", "austrian", "bahasa", "bahasai",
55 "bahasam", "basque", "belarusian", "bosnian", "brazil", "brazilian", "breton", "british",
56 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
57 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "francais",
58 "french", "frenchb", "frenchle", "frenchpro", "friulan", "galician", "german", "germanb",
59 "georgian", "greek", "hebrew", "hungarian", "icelandic", "indon", "indonesian",
60 "interlingua", "irish", "italian", "japanese", "kazakh", "kurmanji", "latin",
61 "latvian", "lithuanian", "lowersorbian", "lsorbian", "macedonian", "magyar", "malay", "meyalu",
62 "mongolian", "naustrian", "newzealand", "ngerman", "ngermanb", "norsk", "nswissgerman",
63 "nynorsk", "piedmontese", "polutonikogreek", "polish", "portuges", "portuguese",
64 "romanian", "romansh", "russian", "russianb", "samin", "scottish", "serbian", "serbian-latin",
65 "slovak", "slovene", "spanish", "swedish", "swissgerman", "thai", "turkish", "turkmen",
66 "ukraineb", "ukrainian", "uppersorbian", "UKenglish", "USenglish", "usorbian",
67 "vietnam", "welsh",
68 0};
69
70 /**
71  * the same as known_languages with .lyx names
72  * please keep this in sync with known_languages line by line!
73  */
74 const char * const known_coded_languages[] = {"french", "afrikaans", "albanian",
75 "american", "arabic_arabi", "arabic_arabtex", "australian", "austrian", "bahasa", "bahasa",
76 "bahasam", "basque", "belarusian", "bosnian", "brazilian", "brazilian", "breton", "british",
77 "bulgarian", "canadian", "canadien", "catalan", "croatian", "czech", "danish",
78 "dutch", "english", "esperanto", "estonian", "farsi", "finnish", "french",
79 "french", "french", "french", "french", "friulan", "galician", "german", "german",
80 "georgian", "greek", "hebrew", "magyar", "icelandic", "bahasa", "bahasa",
81 "interlingua", "irish", "italian", "japanese", "kazakh", "kurmanji", "latin",
82 "latvian", "lithuanian", "lowersorbian", "lowersorbian", "macedonian", "magyar", "bahasam", "bahasam",
83 "mongolian", "naustrian", "newzealand", "ngerman", "ngerman", "norsk", "german-ch",
84 "nynorsk", "piedmontese", "polutonikogreek", "polish", "portuguese", "portuguese",
85 "romanian", "romansh", "russian", "russian", "samin", "scottish", "serbian", "serbian-latin",
86 "slovak", "slovene", "spanish", "swedish", "german-ch-old", "thai", "turkish", "turkmen",
87 "ukrainian", "ukrainian", "uppersorbian", "english", "english", "uppersorbian",
88 "vietnamese", "welsh",
89 0};
90
91 /// languages with british quotes (.lyx names)
92 const char * const known_british_quotes_languages[] = {"british", "welsh", 0};
93
94 /// languages with cjk quotes (.lyx names)
95 const char * const known_cjk_quotes_languages[] = {"chinese-traditional",
96 "japanese", "japanese-cjk", 0};
97
98 /// languages with cjk-angle quotes (.lyx names)
99 const char * const known_cjkangle_quotes_languages[] = {"korean", 0};
100
101 /// languages with danish quotes (.lyx names)
102 const char * const known_danish_quotes_languages[] = {"danish", 0};
103
104 /// languages with english quotes (.lyx names)
105 const char * const known_english_quotes_languages[] = {"american", "australian",
106 "bahasa", "bahasam", "brazilian", "canadian", "chinese-simplified", "english",
107 "esperanto", "farsi", "interlingua", "irish", "newzealand", "scottish",
108 "thai", "turkish", "vietnamese", 0};
109
110 /// languages with french quotes (.lyx names)
111 const char * const known_french_quotes_languages[] = {"ancientgreek",
112 "arabic_arabi", "arabic_arabtex", "asturian", "belarusian", "breton",
113 "canadien", "catalan", "french", "friulan", "galician", "italian", "occitan",
114 "piedmontese", "portuguese", "spanish", "spanish-mexico", 0};
115
116 /// languages with german quotes (.lyx names)
117 const char * const known_german_quotes_languages[] = {"austrian", "bulgarian",
118 "czech", "estonian", "georgian", "german", "icelandic", "latvian", "lithuanian",
119 "lowersorbian", "macedonian", "naustrian", "ngerman", "romansh", "slovak", "slovene",
120 "uppersorbian", 0};
121
122 /// languages with polish quotes (.lyx names)
123 const char * const known_polish_quotes_languages[] = {"afrikaans", "bosnian", "croatian",
124 "dutch", "magyar", "polish", "romanian", "serbian", "serbian-latin", 0};
125
126 /// languages with russian quotes (.lyx names)
127 const char * const known_russian_quotes_languages[] = {"russian", "ukrainian", 0};
128
129 /// languages with swedish quotes (.lyx names)
130 const char * const known_swedish_quotes_languages[] = {"finnish", "swedish", 0};
131
132 /// languages with swiss quotes (.lyx names)
133 const char * const known_swiss_quotes_languages[] = {"albanian",
134 "armenian", "basque", "german-ch", "german-ch-old",
135 "norsk", "nynorsk", "turkmen", "ukrainian", "vietnamese", 0};
136
137 /// known language packages from the times before babel
138 const char * const known_old_language_packages[] = {"french", "frenchle",
139 "frenchpro", "german", "ngerman", "pmfrench", 0};
140
141 char const * const known_fontsizes[] = { "10pt", "11pt", "12pt", 0 };
142
143 const char * const known_roman_fonts[] = { "ae", "beraserif", "bookman",
144 "ccfonts", "chancery", "charter", "cmr", "cochineal", "crimson", "fourier",
145 "garamondx", "libertine", "libertineRoman", "libertine-type1", "lmodern", "mathdesign", "mathpazo",
146 "mathptmx", "MinionPro", "newcent", "NotoSerif-TLF", "PTSerif-TLF", "tgbonum", "tgchorus",
147 "tgpagella", "tgschola", "tgtermes", "utopia", 0 };
148
149 const char * const known_sans_fonts[] = { "avant", "berasans", "biolinum",
150 "biolinum-type1", "cmbr", "cmss", "helvet", "iwona", "iwonac", "iwonal", "iwonalc",
151 "kurier", "kurierc", "kurierl", "kurierlc", "lmss", "NotoSans-TLF", "PTSans-TLF",
152 "tgadventor", "tgheros", "uop", 0 };
153
154 const char * const known_typewriter_fonts[] = { "beramono", "cmtl", "cmtt",
155 "courier", "lmtt", "luximono", "fourier", "libertineMono", "libertineMono-type1", "lmodern",
156 "mathpazo", "mathptmx", "newcent", "NotoMono-TLF", "PTMono-TLF", "tgcursor", "txtt", 0 };
157
158 const char * const known_math_fonts[] = { "eulervm", "newtxmath", 0};
159
160 const char * const known_paper_sizes[] = { "a0paper", "b0paper", "c0paper",
161 "a1paper", "b1paper", "c1paper", "a2paper", "b2paper", "c2paper", "a3paper",
162 "b3paper", "c3paper", "a4paper", "b4paper", "c4paper", "a5paper", "b5paper",
163 "c5paper", "a6paper", "b6paper", "c6paper", "executivepaper", "legalpaper",
164 "letterpaper", "b0j", "b1j", "b2j", "b3j", "b4j", "b5j", "b6j", 0};
165
166 const char * const known_class_paper_sizes[] = { "a4paper", "a5paper",
167 "executivepaper", "legalpaper", "letterpaper", 0};
168
169 const char * const known_paper_margins[] = { "lmargin", "tmargin", "rmargin",
170 "bmargin", "headheight", "headsep", "footskip", "columnsep", 0};
171
172 const char * const known_coded_paper_margins[] = { "leftmargin", "topmargin",
173 "rightmargin", "bottommargin", "headheight", "headsep", "footskip",
174 "columnsep", 0};
175
176 /// commands that can start an \if...\else...\endif sequence
177 const char * const known_if_commands[] = {"if", "ifarydshln", "ifbraket",
178 "ifcancel", "ifcolortbl", "ifeurosym", "ifmarginnote", "ifmmode", "ifpdf",
179 "ifsidecap", "ifupgreek", 0};
180
181 const char * const known_basic_colors[] = {"black", "blue", "brown", "cyan",
182         "darkgray", "gray", "green", "lightgray", "lime", "magenta", "orange", "olive",
183         "pink", "purple", "red", "teal", "violet", "white", "yellow", 0};
184
185 const char * const known_basic_color_codes[] = {"#000000", "#0000ff", "#964B00", "#00ffff",
186         "#a9a9a9", "#808080", "#00ff00", "#d3d3d3", "#bfff00", "#ff00ff", "#ff7f00", "#808000",
187         "#ffc0cb", "#800080", "#ff0000", "#008080", "#8f00ff", "#ffffff", "#ffff00", 0};
188
189 /// conditional commands with three arguments like \@ifundefined{}{}{}
190 const char * const known_if_3arg_commands[] = {"@ifundefined", "IfFileExists",
191 0};
192
193 /*!
194  * Known file extensions for TeX files as used by \\includeonly
195  */
196 char const * const known_tex_extensions[] = {"tex", 0};
197
198 /// packages that work only in xetex
199 /// polyglossia is handled separately
200 const char * const known_xetex_packages[] = {"arabxetex", "fixlatvian",
201 "fontbook", "fontwrap", "mathspec", "philokalia", "unisugar",
202 "xeCJK", "xecolor", "xecyr", "xeindex", "xepersian", "xunicode", 0};
203
204 /// packages that are automatically skipped if loaded by LyX
205 const char * const known_lyx_packages[] = {"amsbsy", "amsmath", "amssymb",
206 "amstext", "amsthm", "array", "babel", "booktabs", "calc", "CJK", "color",
207 "float", "fontspec", "framed", "graphicx", "hhline", "ifthen", "longtable",
208 "makeidx", "minted", "multirow", "nomencl", "pdfpages", "prettyref", "refstyle",
209 "rotating", "rotfloat", "splitidx", "setspace", "subscript", "textcomp", "tipa",
210 "tipx", "tone", "ulem", "url", "varioref", "verbatim", "wrapfig", "xcolor",
211 "xunicode", 0};
212
213 // codes used to remove packages that are loaded automatically by LyX.
214 // Syntax: package_beg_sep<name>package_mid_sep<package loading code>package_end_sep
215 const char package_beg_sep = '\001';
216 const char package_mid_sep = '\002';
217 const char package_end_sep = '\003';
218
219
220 // returns true if at least one of the options in what has been found
221 bool handle_opt(vector<string> & opts, char const * const * what, string & target)
222 {
223         if (opts.empty())
224                 return false;
225
226         bool found = false;
227         // the last language option is the document language (for babel and LyX)
228         // the last size option is the document font size
229         vector<string>::iterator it;
230         vector<string>::iterator position = opts.begin();
231         for (; *what; ++what) {
232                 it = find(opts.begin(), opts.end(), *what);
233                 if (it != opts.end()) {
234                         if (it >= position) {
235                                 found = true;
236                                 target = *what;
237                                 position = it;
238                         }
239                 }
240         }
241         return found;
242 }
243
244
245 void delete_opt(vector<string> & opts, char const * const * what)
246 {
247         if (opts.empty())
248                 return;
249
250         // remove found options from the list
251         // do this after handle_opt to avoid potential memory leaks
252         vector<string>::iterator it;
253         for (; *what; ++what) {
254                 it = find(opts.begin(), opts.end(), *what);
255                 if (it != opts.end())
256                         opts.erase(it);
257         }
258 }
259
260
261 /*!
262  * Split a package options string (keyval format) into a vector.
263  * Example input:
264  *   authorformat=smallcaps,
265  *   commabeforerest,
266  *   titleformat=colonsep,
267  *   bibformat={tabular,ibidem,numbered}
268  */
269 vector<string> split_options(string const & input)
270 {
271         vector<string> options;
272         string option;
273         Parser p(input);
274         while (p.good()) {
275                 Token const & t = p.get_token();
276                 if (t.asInput() == ",") {
277                         options.push_back(trimSpaceAndEol(option));
278                         option.erase();
279                 } else if (t.asInput() == "=") {
280                         option += '=';
281                         p.skip_spaces(true);
282                         if (p.next_token().asInput() == "{")
283                                 option += '{' + p.getArg('{', '}') + '}';
284                 } else if (t.cat() != catSpace && t.cat() != catComment)
285                         option += t.asInput();
286         }
287
288         if (!option.empty())
289                 options.push_back(trimSpaceAndEol(option));
290
291         return options;
292 }
293
294
295 /*!
296  * Retrieve a keyval option "name={value with=sign}" named \p name from
297  * \p options and return the value.
298  * The found option is also removed from \p options.
299  */
300 string process_keyval_opt(vector<string> & options, string name)
301 {
302         for (size_t i = 0; i < options.size(); ++i) {
303                 vector<string> option;
304                 split(options[i], option, '=');
305                 if (option.size() < 2)
306                         continue;
307                 if (option[0] == name) {
308                         options.erase(options.begin() + i);
309                         option.erase(option.begin());
310                         return join(option, "=");
311                 }
312         }
313         return "";
314 }
315
316 } // anonymous namespace
317
318
319 /**
320  * known polyglossia language names (including variants)
321  * FIXME: support spelling=old for german variants (german vs. ngerman LyX names etc)
322  */
323 const char * const Preamble::polyglossia_languages[] = {
324 "albanian", "american", "amharic", "ancient", "arabic", "armenian", "asturian", "australian",
325 "bahasai", "bahasam", "basque", "bengali", "brazil", "brazilian", "breton", "british", "bulgarian",
326 "catalan", "coptic", "croatian", "czech", "danish", "divehi", "dutch",
327 "english", "esperanto", "estonian", "farsi", "finnish", "french", "friulan",
328 "galician", "greek", "monotonic", "hebrew", "hindi",
329 "icelandic", "interlingua", "irish", "italian", "kannada", "khmer",
330 "lao", "latin", "latvian", "lithuanian", "lsorbian", "magyar", "malayalam", "marathi",
331 "austrian", "newzealand", "german", "norsk", "nynorsk", "occitan",
332 "piedmontese", "polish", "polytonic", "portuges", "romanian", "romansh", "russian",
333 "samin", "sanskrit", "scottish", "serbian", "slovak", "slovenian", "spanish", "swedish", "syriac",
334 "tamil", "telugu", "thai", "tibetan", "turkish", "turkmen",
335 "ukrainian", "urdu", "usorbian", "vietnamese", "welsh", 0};
336 // not yet supported by LyX: "korean", "nko"
337
338 /**
339  * the same as polyglossia_languages with .lyx names
340  * please keep this in sync with polyglossia_languages line by line!
341  */
342 const char * const Preamble::coded_polyglossia_languages[] = {
343 "albanian", "american", "amharic", "ancientgreek", "arabic_arabi", "armenian", "asturian", "australian",
344 "bahasa", "bahasam", "basque", "bengali", "brazilian", "brazilian", "breton", "british", "bulgarian",
345 "catalan", "coptic", "croatian", "czech", "danish", "divehi", "dutch",
346 "english", "esperanto", "estonian", "farsi", "finnish", "french", "friulan",
347 "galician", "greek", "greek", "hebrew", "hindi",
348 "icelandic", "interlingua", "irish", "italian", "kannada", "khmer",
349 "lao", "latin", "latvian", "lithuanian", "lowersorbian", "magyar", "malayalam", "marathi",
350 "naustrian","newzealand", "ngerman", "norsk", "nynorsk", "occitan",
351 "piedmontese", "polish", "polutonikogreek", "portuges", "romanian", "romansh", "russian",
352 "samin", "sanskrit", "scottish", "serbian", "slovak", "slovene", "spanish", "swedish", "syriac",
353 "tamil", "telugu", "thai", "tibetan", "turkish", "turkmen",
354 "ukrainian", "urdu", "uppersorbian", "vietnamese", "welsh", 0};
355 // not yet supported by LyX: "korean-polyglossia", "nko"
356
357
358 bool Preamble::usePolyglossia() const
359 {
360         return h_use_non_tex_fonts && h_language_package == "default";
361 }
362
363
364 bool Preamble::indentParagraphs() const
365 {
366         return h_paragraph_separation == "indent";
367 }
368
369
370 bool Preamble::isPackageUsed(string const & package) const
371 {
372         return used_packages.find(package) != used_packages.end();
373 }
374
375
376 vector<string> Preamble::getPackageOptions(string const & package) const
377 {
378         map<string, vector<string> >::const_iterator it = used_packages.find(package);
379         if (it != used_packages.end())
380                 return it->second;
381         return vector<string>();
382 }
383
384
385 void Preamble::registerAutomaticallyLoadedPackage(std::string const & package)
386 {
387         auto_packages.insert(package);
388 }
389
390
391 void Preamble::addModule(string const & module)
392 {
393         used_modules.push_back(module);
394 }
395
396
397 void Preamble::suppressDate(bool suppress)
398 {
399         if (suppress)
400                 h_suppress_date = "true";
401         else
402                 h_suppress_date = "false";
403 }
404
405
406 void Preamble::registerAuthor(std::string const & name)
407 {
408         Author author(from_utf8(name), empty_docstring());
409         author.setUsed(true);
410         authors_.record(author);
411         h_tracking_changes = "true";
412         h_output_changes = "true";
413 }
414
415
416 Author const & Preamble::getAuthor(std::string const & name) const
417 {
418         Author author(from_utf8(name), empty_docstring());
419         for (AuthorList::Authors::const_iterator it = authors_.begin();
420              it != authors_.end(); ++it)
421                 if (*it == author)
422                         return *it;
423         static Author const dummy;
424         return dummy;
425 }
426
427
428 int Preamble::getSpecialTableColumnArguments(char c) const
429 {
430         map<char, int>::const_iterator it = special_columns_.find(c);
431         if (it == special_columns_.end())
432                 return -1;
433         return it->second;
434 }
435
436
437 void Preamble::add_package(string const & name, vector<string> & options)
438 {
439         // every package inherits the global options
440         if (used_packages.find(name) == used_packages.end())
441                 used_packages[name] = split_options(h_options);
442
443         // Insert options passed via PassOptionsToPackage
444         for (auto const & p : extra_package_options_) {
445                 if (p.first == name) {
446                         vector<string> eo = getVectorFromString(p.second);
447                         for (auto const & eoi : eo)
448                                 options.push_back(eoi);
449                 }
450
451         }
452         vector<string> & v = used_packages[name];
453         v.insert(v.end(), options.begin(), options.end());
454         if (name == "jurabib") {
455                 // Don't output the order argument (see the cite command
456                 // handling code in text.cpp).
457                 vector<string>::iterator end =
458                         remove(options.begin(), options.end(), "natbiborder");
459                 end = remove(options.begin(), end, "jurabiborder");
460                 options.erase(end, options.end());
461         }
462 }
463
464
465 namespace {
466
467 // Given is a string like "scaled=0.9" or "Scale=0.9", return 0.9 * 100
468 bool scale_as_percentage(string const & scale, string & percentage)
469 {
470         string::size_type pos = scale.find('=');
471         if (pos != string::npos) {
472                 string value = scale.substr(pos + 1);
473                 if (isStrDbl(value)) {
474                         percentage = convert<string>(
475                                 static_cast<int>(100 * convert<double>(value)));
476                         return true;
477                 }
478         }
479         return false;
480 }
481
482
483 string remove_braces(string const & value)
484 {
485         if (value.empty())
486                 return value;
487         if (value[0] == '{' && value[value.length()-1] == '}')
488                 return value.substr(1, value.length()-2);
489         return value;
490 }
491
492 } // anonymous namespace
493
494
495 Preamble::Preamble() : one_language(true), explicit_babel(false),
496         title_layout_found(false), index_number(0), h_font_cjk_set(false),
497         h_use_microtype("false")
498 {
499         //h_backgroundcolor;
500         //h_boxbgcolor;
501         h_biblio_style            = "plain";
502         h_bibtex_command          = "default";
503         h_cite_engine             = "basic";
504         h_cite_engine_type        = "default";
505         h_color                   = "#008000";
506         h_defskip                 = "medskip";
507         h_dynamic_quotes          = false;
508         //h_float_placement;
509         //h_fontcolor;
510         h_fontencoding            = "default";
511         h_font_roman[0]           = "default";
512         h_font_roman[1]           = "default";
513         h_font_sans[0]            = "default";
514         h_font_sans[1]            = "default";
515         h_font_typewriter[0]      = "default";
516         h_font_typewriter[1]      = "default";
517         h_font_math[0]            = "auto";
518         h_font_math[1]            = "auto";
519         h_font_default_family     = "default";
520         h_use_non_tex_fonts       = false;
521         h_font_sc                 = "false";
522         h_font_osf                = "false";
523         h_font_sf_scale[0]        = "100";
524         h_font_sf_scale[1]        = "100";
525         h_font_tt_scale[0]        = "100";
526         h_font_tt_scale[1]        = "100";
527         //h_font_cjk
528         h_is_mathindent           = "0";
529         h_math_numbering_side     = "default";
530         h_graphics                = "default";
531         h_default_output_format   = "default";
532         h_html_be_strict          = "false";
533         h_html_css_as_file        = "0";
534         h_html_math_output        = "0";
535         h_index[0]                = "Index";
536         h_index_command           = "default";
537         h_inputencoding           = "auto";
538         h_justification           = "true";
539         h_language                = "english";
540         h_language_package        = "none";
541         //h_listings_params;
542         h_maintain_unincluded_children = "false";
543         //h_margins;
544         //h_notefontcolor;
545         //h_options;
546         h_output_changes          = "false";
547         h_output_sync             = "0";
548         //h_output_sync_macro
549         h_papercolumns            = "1";
550         h_paperfontsize           = "default";
551         h_paperorientation        = "portrait";
552         h_paperpagestyle          = "default";
553         //h_papersides;
554         h_papersize               = "default";
555         h_paragraph_indentation   = "default";
556         h_paragraph_separation    = "indent";
557         //h_pdf_title;
558         //h_pdf_author;
559         //h_pdf_subject;
560         //h_pdf_keywords;
561         h_pdf_bookmarks           = "0";
562         h_pdf_bookmarksnumbered   = "0";
563         h_pdf_bookmarksopen       = "0";
564         h_pdf_bookmarksopenlevel  = "1";
565         h_pdf_breaklinks          = "0";
566         h_pdf_pdfborder           = "0";
567         h_pdf_colorlinks          = "0";
568         h_pdf_backref             = "section";
569         h_pdf_pdfusetitle         = "0";
570         //h_pdf_pagemode;
571         //h_pdf_quoted_options;
572         h_quotes_style         = "english";
573         h_secnumdepth             = "3";
574         h_shortcut[0]             = "idx";
575         h_spacing                 = "single";
576         h_save_transient_properties = "true";
577         h_suppress_date           = "false";
578         h_textclass               = "article";
579         h_tocdepth                = "3";
580         h_tracking_changes        = "false";
581         h_use_bibtopic            = "false";
582         h_use_dash_ligatures      = "true";
583         h_use_indices             = "false";
584         h_use_geometry            = "false";
585         h_use_default_options     = "false";
586         h_use_hyperref            = "false";
587         h_use_microtype           = "false";
588         h_use_refstyle            = false;
589         h_use_minted              = false;
590         h_use_packages["amsmath"]    = "1";
591         h_use_packages["amssymb"]    = "0";
592         h_use_packages["cancel"]     = "0";
593         h_use_packages["esint"]      = "1";
594         h_use_packages["mhchem"]     = "0";
595         h_use_packages["mathdots"]   = "0";
596         h_use_packages["mathtools"]  = "0";
597         h_use_packages["stackrel"]   = "0";
598         h_use_packages["stmaryrd"]   = "0";
599         h_use_packages["undertilde"] = "0";
600 }
601
602
603 void Preamble::handle_hyperref(vector<string> & options)
604 {
605         // FIXME swallow inputencoding changes that might surround the
606         //       hyperref setup if it was written by LyX
607         h_use_hyperref = "true";
608         // swallow "unicode=true", since LyX does always write that
609         vector<string>::iterator it =
610                 find(options.begin(), options.end(), "unicode=true");
611         if (it != options.end())
612                 options.erase(it);
613         it = find(options.begin(), options.end(), "pdfusetitle");
614         if (it != options.end()) {
615                 h_pdf_pdfusetitle = "1";
616                 options.erase(it);
617         }
618         string bookmarks = process_keyval_opt(options, "bookmarks");
619         if (bookmarks == "true")
620                 h_pdf_bookmarks = "1";
621         else if (bookmarks == "false")
622                 h_pdf_bookmarks = "0";
623         if (h_pdf_bookmarks == "1") {
624                 string bookmarksnumbered =
625                         process_keyval_opt(options, "bookmarksnumbered");
626                 if (bookmarksnumbered == "true")
627                         h_pdf_bookmarksnumbered = "1";
628                 else if (bookmarksnumbered == "false")
629                         h_pdf_bookmarksnumbered = "0";
630                 string bookmarksopen =
631                         process_keyval_opt(options, "bookmarksopen");
632                 if (bookmarksopen == "true")
633                         h_pdf_bookmarksopen = "1";
634                 else if (bookmarksopen == "false")
635                         h_pdf_bookmarksopen = "0";
636                 if (h_pdf_bookmarksopen == "1") {
637                         string bookmarksopenlevel =
638                                 process_keyval_opt(options, "bookmarksopenlevel");
639                         if (!bookmarksopenlevel.empty())
640                                 h_pdf_bookmarksopenlevel = bookmarksopenlevel;
641                 }
642         }
643         string breaklinks = process_keyval_opt(options, "breaklinks");
644         if (breaklinks == "true")
645                 h_pdf_breaklinks = "1";
646         else if (breaklinks == "false")
647                 h_pdf_breaklinks = "0";
648         string pdfborder = process_keyval_opt(options, "pdfborder");
649         if (pdfborder == "{0 0 0}")
650                 h_pdf_pdfborder = "1";
651         else if (pdfborder == "{0 0 1}")
652                 h_pdf_pdfborder = "0";
653         string backref = process_keyval_opt(options, "backref");
654         if (!backref.empty())
655                 h_pdf_backref = backref;
656         string colorlinks = process_keyval_opt(options, "colorlinks");
657         if (colorlinks == "true")
658                 h_pdf_colorlinks = "1";
659         else if (colorlinks == "false")
660                 h_pdf_colorlinks = "0";
661         string pdfpagemode = process_keyval_opt(options, "pdfpagemode");
662         if (!pdfpagemode.empty())
663                 h_pdf_pagemode = pdfpagemode;
664         string pdftitle = process_keyval_opt(options, "pdftitle");
665         if (!pdftitle.empty()) {
666                 h_pdf_title = remove_braces(pdftitle);
667         }
668         string pdfauthor = process_keyval_opt(options, "pdfauthor");
669         if (!pdfauthor.empty()) {
670                 h_pdf_author = remove_braces(pdfauthor);
671         }
672         string pdfsubject = process_keyval_opt(options, "pdfsubject");
673         if (!pdfsubject.empty())
674                 h_pdf_subject = remove_braces(pdfsubject);
675         string pdfkeywords = process_keyval_opt(options, "pdfkeywords");
676         if (!pdfkeywords.empty())
677                 h_pdf_keywords = remove_braces(pdfkeywords);
678         if (!options.empty()) {
679                 if (!h_pdf_quoted_options.empty())
680                         h_pdf_quoted_options += ',';
681                 h_pdf_quoted_options += join(options, ",");
682                 options.clear();
683         }
684 }
685
686
687 void Preamble::handle_geometry(vector<string> & options)
688 {
689         h_use_geometry = "true";
690         vector<string>::iterator it;
691         // paper orientation
692         if ((it = find(options.begin(), options.end(), "landscape")) != options.end()) {
693                 h_paperorientation = "landscape";
694                 options.erase(it);
695         }
696         // paper size
697         // keyval version: "paper=letter"
698         string paper = process_keyval_opt(options, "paper");
699         if (!paper.empty())
700                 h_papersize = paper + "paper";
701         // alternative version: "letterpaper"
702         handle_opt(options, known_paper_sizes, h_papersize);
703         delete_opt(options, known_paper_sizes);
704         // page margins
705         char const * const * margin = known_paper_margins;
706         for (; *margin; ++margin) {
707                 string value = process_keyval_opt(options, *margin);
708                 if (!value.empty()) {
709                         int k = margin - known_paper_margins;
710                         string name = known_coded_paper_margins[k];
711                         h_margins += '\\' + name + ' ' + value + '\n';
712                 }
713         }
714 }
715
716
717 void Preamble::handle_package(Parser &p, string const & name,
718                               string const & opts, bool in_lyx_preamble,
719                               bool detectEncoding)
720 {
721         vector<string> options = split_options(opts);
722         add_package(name, options);
723
724         if (is_known(name, known_xetex_packages)) {
725                 xetex = true;
726                 h_use_non_tex_fonts = true;
727                 registerAutomaticallyLoadedPackage("fontspec");
728                 if (h_inputencoding == "auto")
729                         p.setEncoding("UTF-8");
730         }
731
732         // roman fonts
733         if (is_known(name, known_roman_fonts))
734                 h_font_roman[0] = name;
735
736         if (name == "fourier") {
737                 h_font_roman[0] = "utopia";
738                 // when font uses real small capitals
739                 if (opts == "expert")
740                         h_font_sc = "true";
741         }
742
743         if (name == "garamondx") {
744                 h_font_roman[0] = "garamondx";
745                 if (opts == "osfI")
746                         h_font_osf = "true";
747         }
748
749         if (name == "libertine") {
750                 h_font_roman[0] = "libertine";
751                 // this automatically invokes biolinum
752                 h_font_sans[0] = "biolinum";
753                 // as well as libertineMono
754                 h_font_typewriter[0] = "libertine-mono";
755                 if (opts == "osf")
756                         h_font_osf = "true";
757                 else if (opts == "lining")
758                         h_font_osf = "false";
759         }
760
761         if (name == "libertineRoman" || name == "libertine-type1") {
762                 h_font_roman[0] = "libertine";
763                 // NOTE: contrary to libertine.sty, libertineRoman
764                 // and libertine-type1 do not automatically invoke
765                 // biolinum and libertineMono
766                 if (opts == "lining")
767                         h_font_osf = "false";
768                 else if (opts == "osf")
769                         h_font_osf = "true";
770         }
771
772         if (name == "MinionPro") {
773                 h_font_roman[0] = "minionpro";
774                 if (opts.find("lf") != string::npos)
775                         h_font_osf = "false";
776                 else
777                         h_font_osf = "true";
778                 if (opts.find("onlytext") != string::npos)
779                         h_font_math[0] = "default";
780                 else
781                         h_font_math[0] = "auto";
782         }
783
784         if (name == "mathdesign") {
785                 if (opts.find("charter") != string::npos)
786                         h_font_roman[0] = "md-charter";
787                 if (opts.find("garamond") != string::npos)
788                         h_font_roman[0] = "md-garamond";
789                 if (opts.find("utopia") != string::npos)
790                         h_font_roman[0] = "md-utopia";
791                 if (opts.find("expert") != string::npos) {
792                         h_font_sc = "true";
793                         h_font_osf = "true";
794                 }
795         }
796
797         else if (name == "mathpazo")
798                 h_font_roman[0] = "palatino";
799
800         else if (name == "mathptmx")
801                 h_font_roman[0] = "times";
802
803         if (name == "crimson")
804                 h_font_roman[0] = "cochineal";
805
806         if (name == "cochineal") {
807                 h_font_roman[0] = "cochineal";
808                 // cochineal can have several options, e.g. [proportional,osf]
809                 string::size_type pos = opts.find("osf");
810                 if (pos != string::npos)
811                         h_font_osf = "true";
812         }
813
814         if (name == "noto") {
815                 // noto can have several options
816                 if (opts.empty())
817                         h_font_roman[0] = "NotoSerif-TLF";
818                 string::size_type pos = opts.find("rm");
819                 if (pos != string::npos)
820                         h_font_roman[0] = "NotoSerif-TLF";
821                 pos = opts.find("sf");
822                 if (pos != string::npos)
823                         h_font_sans[0] = "NotoSans-TLF";
824                 pos = opts.find("nott");
825                 if (pos != string::npos) {
826                         h_font_roman[0] = "NotoSerif-TLF";
827                         h_font_sans[0] = "NotoSans-TLF";
828                 }
829                 // noto as typewriter is handled in handling of \ttdefault
830                 // special cases are handled in handling of \rmdefault and \sfdefault
831         }
832
833         if (name == "paratype") {
834                 // in this case all fonts are ParaType
835                 h_font_roman[0] = "PTSerif-TLF";
836                 h_font_sans[0] = "default";
837                 h_font_typewriter[0] = "default";
838         }
839
840         if (name == "PTSerif")
841                 h_font_roman[0] = "PTSerif-TLF";
842
843         // sansserif fonts
844         if (is_known(name, known_sans_fonts)) {
845                 h_font_sans[0] = name;
846                 if (options.size() >= 1) {
847                         if (scale_as_percentage(opts, h_font_sf_scale[0]))
848                                 options.clear();
849                 }
850         }
851
852         if (name == "biolinum" || name == "biolinum-type1") {
853                 h_font_sans[0] = "biolinum";
854                 // biolinum can have several options, e.g. [osf,scaled=0.97]
855                 string::size_type pos = opts.find("osf");
856                 if (pos != string::npos)
857                         h_font_osf = "true";
858         }
859
860         if (name == "PTSans") {
861                 h_font_sans[0] = "PTSans-TLF";
862                 if (options.size() >= 1) {
863                         if (scale_as_percentage(opts, h_font_sf_scale[0]))
864                                 options.clear();
865                 }
866         }
867
868         // typewriter fonts
869         if (is_known(name, known_typewriter_fonts)) {
870                 // fourier can be set as roman font _only_
871                 // fourier as typewriter is handled in handling of \ttdefault
872                 if (name != "fourier") {
873                         h_font_typewriter[0] = name;
874                         if (options.size() >= 1) {
875                                 if (scale_as_percentage(opts, h_font_tt_scale[0]))
876                                         options.clear();
877                         }
878                 }
879         }
880
881         if (name == "libertineMono" || name == "libertineMono-type1")
882                 h_font_typewriter[0] = "libertine-mono";
883
884         if (name == "PTMono") {
885                 h_font_typewriter[0] = "PTMono-TLF";
886                 if (options.size() >= 1) {
887                         if (scale_as_percentage(opts, h_font_tt_scale[0]))
888                                 options.clear();
889                 }
890         }
891
892         // font uses old-style figure
893         if (name == "eco")
894                 h_font_osf = "true";
895
896         // math fonts
897         if (is_known(name, known_math_fonts))
898                 h_font_math[0] = name;
899
900         if (name == "newtxmath") {
901                 if (opts.empty())
902                         h_font_math[0] = "newtxmath";
903                 else if (opts == "garamondx")
904                         h_font_math[0] = "garamondx-ntxm";
905                 else if (opts == "libertine")
906                         h_font_math[0] = "libertine-ntxm";
907                 else if (opts == "minion")
908                         h_font_math[0] = "minion-ntxm";
909                 else if (opts == "cochineal")
910                         h_font_math[0] = "cochineal-ntxm";
911         }
912
913         if (name == "iwona")
914                 if (opts == "math")
915                         h_font_math[0] = "iwona-math";
916
917         if (name == "kurier")
918                 if (opts == "math")
919                         h_font_math[0] = "kurier-math";
920
921         // after the detection and handling of special cases, we can remove the
922         // fonts, otherwise they would appear in the preamble, see bug #7856
923         if (is_known(name, known_roman_fonts) || is_known(name, known_sans_fonts)
924                 ||      is_known(name, known_typewriter_fonts) || is_known(name, known_math_fonts))
925                 ;
926         //"On". See the enum Package in BufferParams.h if you thought that "2" should have been "42"
927         else if (name == "amsmath" || name == "amssymb" || name == "cancel" ||
928                  name == "esint" || name == "mhchem" || name == "mathdots" ||
929                  name == "mathtools" || name == "stackrel" ||
930                  name == "stmaryrd" || name == "undertilde")
931                 h_use_packages[name] = "2";
932
933         else if (name == "babel") {
934                 h_language_package = "default";
935                 // One might think we would have to do nothing if babel is loaded
936                 // without any options to prevent pollution of the preamble with this
937                 // babel call in every roundtrip.
938                 // But the user could have defined babel-specific things afterwards. So
939                 // we need to keep it in the preamble to prevent cases like bug #7861.
940                 if (!opts.empty()) {
941                         // check if more than one option was used - used later for inputenc
942                         if (options.begin() != options.end() - 1)
943                                 one_language = false;
944                         // babel takes the last language of the option of its \usepackage
945                         // call as document language. If there is no such language option, the
946                         // last language in the documentclass options is used.
947                         handle_opt(options, known_languages, h_language);
948                         // translate the babel name to a LyX name
949                         h_language = babel2lyx(h_language);
950                         if (h_language == "japanese") {
951                                 // For Japanese, the encoding isn't indicated in the source
952                                 // file, and there's really not much we can do. We could
953                                 // 1) offer a list of possible encodings to choose from, or
954                                 // 2) determine the encoding of the file by inspecting it.
955                                 // For the time being, we leave the encoding alone so that
956                                 // we don't get iconv errors when making a wrong guess, and
957                                 // we will output a note at the top of the document
958                                 // explaining what to do.
959                                 Encoding const * const enc = encodings.fromIconvName(
960                                         p.getEncoding(), Encoding::japanese, false);
961                                 if (enc)
962                                         h_inputencoding = enc->name();
963                                 is_nonCJKJapanese = true;
964                                 // in this case babel can be removed from the preamble
965                                 registerAutomaticallyLoadedPackage("babel");
966                         } else {
967                                 // If babel is called with options, LyX puts them by default into the
968                                 // document class options. This works for most languages, except
969                                 // for Latvian, Lithuanian, Mongolian, Turkmen and Vietnamese and
970                                 // perhaps in future others.
971                                 // Therefore keep the babel call as it is as the user might have
972                                 // reasons for it.
973                                 h_preamble << "\\usepackage[" << opts << "]{babel}\n";
974                         }
975                         delete_opt(options, known_languages);
976                 } else {
977                         h_preamble << "\\usepackage{babel}\n";
978                         explicit_babel = true;
979                 }
980         }
981
982         else if (name == "polyglossia") {
983                 h_language_package = "default";
984                 h_default_output_format = "pdf4";
985                 h_use_non_tex_fonts = true;
986                 xetex = true;
987                 registerAutomaticallyLoadedPackage("xunicode");
988                 if (h_inputencoding == "auto")
989                         p.setEncoding("UTF-8");
990         }
991
992         else if (name == "CJK") {
993                 // set the encoding to "auto" because it might be set to "default" by the babel handling
994                 // and this would not be correct for CJK
995                 if (h_inputencoding == "default")
996                         h_inputencoding = "auto";
997                 registerAutomaticallyLoadedPackage("CJK");
998         }
999
1000         else if (name == "CJKutf8") {
1001                 h_inputencoding = "utf8-cjk";
1002                 p.setEncoding("UTF-8");
1003                 registerAutomaticallyLoadedPackage("CJKutf8");
1004         }
1005
1006         else if (name == "fontenc") {
1007                 h_fontencoding = getStringFromVector(options, ",");
1008                 /* We could do the following for better round trip support,
1009                  * but this makes the document less portable, so I skip it:
1010                 if (h_fontencoding == lyxrc.fontenc)
1011                         h_fontencoding = "global";
1012                  */
1013                 options.clear();
1014         }
1015
1016         else if (name == "inputenc" || name == "luainputenc") {
1017                 // h_inputencoding is only set when there is not more than one
1018                 // inputenc option because otherwise h_inputencoding must be
1019                 // set to "auto" (the default encoding of the document language)
1020                 // Therefore check that exactly one option is passed to inputenc.
1021                 // It is also only set when there is not more than one babel
1022                 // language option.
1023                 if (!options.empty()) {
1024                         string const encoding = options.back();
1025                         Encoding const * const enc = encodings.fromLaTeXName(
1026                                 encoding, Encoding::inputenc, true);
1027                         if (!enc) {
1028                                 if (!detectEncoding)
1029                                         cerr << "Unknown encoding " << encoding
1030                                              << ". Ignoring." << std::endl;
1031                         } else {
1032                                 if (!enc->unsafe() && options.size() == 1 && one_language == true)
1033                                         h_inputencoding = enc->name();
1034                                 p.setEncoding(enc->iconvName());
1035                         }
1036                         options.clear();
1037                 }
1038         }
1039
1040         else if (name == "srcltx") {
1041                 h_output_sync = "1";
1042                 if (!opts.empty()) {
1043                         h_output_sync_macro = "\\usepackage[" + opts + "]{srcltx}";
1044                         options.clear();
1045                 } else
1046                         h_output_sync_macro = "\\usepackage{srcltx}";
1047         }
1048
1049         else if (is_known(name, known_old_language_packages)) {
1050                 // known language packages from the times before babel
1051                 // if they are found and not also babel, they will be used as
1052                 // custom language package
1053                 h_language_package = "\\usepackage{" + name + "}";
1054         }
1055
1056         else if (name == "lyxskak") {
1057                 // ignore this and its options
1058                 const char * const o[] = {"ps", "mover", 0};
1059                 delete_opt(options, o);
1060         }
1061
1062         else if (is_known(name, known_lyx_packages) && options.empty()) {
1063                 if (name == "splitidx")
1064                         h_use_indices = "true";
1065                 else if (name == "minted")
1066                         h_use_minted = true;
1067                 else if (name == "refstyle")
1068                         h_use_refstyle = true;
1069                 else if (name == "prettyref")
1070                         h_use_refstyle = false;
1071                 if (!in_lyx_preamble) {
1072                         h_preamble << package_beg_sep << name
1073                                    << package_mid_sep << "\\usepackage{"
1074                                    << name << '}';
1075                         if (p.next_token().cat() == catNewline ||
1076                             (p.next_token().cat() == catSpace &&
1077                              p.next_next_token().cat() == catNewline))
1078                                 h_preamble << '\n';
1079                         h_preamble << package_end_sep;
1080                 }
1081         }
1082
1083         else if (name == "geometry")
1084                 handle_geometry(options);
1085
1086         else if (name == "subfig")
1087                 ; // ignore this FIXME: Use the package separator mechanism instead
1088
1089         else if (char const * const * where = is_known(name, known_languages))
1090                 h_language = known_coded_languages[where - known_languages];
1091
1092         else if (name == "natbib") {
1093                 h_biblio_style = "plainnat";
1094                 h_cite_engine = "natbib";
1095                 h_cite_engine_type = "authoryear";
1096                 vector<string>::iterator it =
1097                         find(options.begin(), options.end(), "authoryear");
1098                 if (it != options.end())
1099                         options.erase(it);
1100                 else {
1101                         it = find(options.begin(), options.end(), "numbers");
1102                         if (it != options.end()) {
1103                                 h_cite_engine_type = "numerical";
1104                                 options.erase(it);
1105                         }
1106                 }
1107                 if (!options.empty())
1108                         h_biblio_options = join(options, ",");
1109         }
1110
1111         else if (name == "biblatex") {
1112                 h_biblio_style = "plainnat";
1113                 h_cite_engine = "biblatex";
1114                 h_cite_engine_type = "authoryear";
1115                 string opt;
1116                 vector<string>::iterator it =
1117                         find(options.begin(), options.end(), "natbib");
1118                 if (it != options.end()) {
1119                         options.erase(it);
1120                         h_cite_engine = "biblatex-natbib";
1121                 } else {
1122                         opt = process_keyval_opt(options, "natbib");
1123                         if (opt == "true")
1124                                 h_cite_engine = "biblatex-natbib";
1125                 }
1126                 opt = process_keyval_opt(options, "style");
1127                 if (!opt.empty()) {
1128                         h_biblatex_citestyle = opt;
1129                         h_biblatex_bibstyle = opt;
1130                 } else {
1131                         opt = process_keyval_opt(options, "citestyle");
1132                         if (!opt.empty())
1133                                 h_biblatex_citestyle = opt;
1134                         opt = process_keyval_opt(options, "bibstyle");
1135                         if (!opt.empty())
1136                                 h_biblatex_bibstyle = opt;
1137                 }
1138                 opt = process_keyval_opt(options, "refsection");
1139                 if (!opt.empty()) {
1140                         if (opt == "none" || opt == "part"
1141                             || opt == "chapter" || opt == "section"
1142                             || opt == "subsection")
1143                                 h_multibib = opt;
1144                         else
1145                                 cerr << "Ignoring unkown refesection value '"
1146                                      << opt << "'.";
1147                 }
1148                 if (!options.empty()) {
1149                         h_biblio_options = join(options, ",");
1150                         options.clear();
1151                 }
1152         }
1153
1154         else if (name == "jurabib") {
1155                 h_biblio_style = "jurabib";
1156                 h_cite_engine = "jurabib";
1157                 h_cite_engine_type = "authoryear";
1158                 if (!options.empty())
1159                         h_biblio_options = join(options, ",");
1160         }
1161
1162         else if (name == "bibtopic")
1163                 h_use_bibtopic = "true";
1164
1165         else if (name == "chapterbib")
1166                 h_multibib = "child";
1167
1168         else if (name == "hyperref")
1169                 handle_hyperref(options);
1170
1171         else if (name == "algorithm2e") {
1172                 // Load "algorithm2e" module
1173                 addModule("algorithm2e");
1174                 // Add the package options to the global document options
1175                 if (!options.empty()) {
1176                         if (h_options.empty())
1177                                 h_options = join(options, ",");
1178                         else
1179                                 h_options += ',' + join(options, ",");
1180                 }
1181         }
1182         else if (name == "microtype") {
1183                 //we internally support only microtype without params
1184                 if (options.empty())
1185                         h_use_microtype = "true";
1186                 else
1187                         h_preamble << "\\usepackage[" << opts << "]{microtype}";
1188         }
1189
1190         else if (!in_lyx_preamble) {
1191                 if (options.empty())
1192                         h_preamble << "\\usepackage{" << name << '}';
1193                 else {
1194                         h_preamble << "\\usepackage[" << opts << "]{"
1195                                    << name << '}';
1196                         options.clear();
1197                 }
1198                 if (p.next_token().cat() == catNewline ||
1199                     (p.next_token().cat() == catSpace &&
1200                      p.next_next_token().cat() == catNewline))
1201                         h_preamble << '\n';
1202         }
1203
1204         // We need to do something with the options...
1205         if (!options.empty() && !detectEncoding)
1206                 cerr << "Ignoring options '" << join(options, ",")
1207                      << "' of package " << name << '.' << endl;
1208
1209         // remove the whitespace
1210         p.skip_spaces();
1211 }
1212
1213
1214 void Preamble::handle_if(Parser & p, bool in_lyx_preamble)
1215 {
1216         while (p.good()) {
1217                 Token t = p.get_token();
1218                 if (t.cat() == catEscape &&
1219                     is_known(t.cs(), known_if_commands))
1220                         handle_if(p, in_lyx_preamble);
1221                 else {
1222                         if (!in_lyx_preamble)
1223                                 h_preamble << t.asInput();
1224                         if (t.cat() == catEscape && t.cs() == "fi")
1225                                 return;
1226                 }
1227         }
1228 }
1229
1230
1231 bool Preamble::writeLyXHeader(ostream & os, bool subdoc, string const & outfiledir)
1232 {
1233         if (contains(h_float_placement, "H"))
1234                 registerAutomaticallyLoadedPackage("float");
1235         if (h_spacing != "single" && h_spacing != "default")
1236                 registerAutomaticallyLoadedPackage("setspace");
1237         if (h_use_packages["amsmath"] == "2") {
1238                 // amsbsy and amstext are already provided by amsmath
1239                 registerAutomaticallyLoadedPackage("amsbsy");
1240                 registerAutomaticallyLoadedPackage("amstext");
1241         }
1242
1243         // output the LyX file settings
1244         // Important: Keep the version formatting in sync with LyX and
1245         //            lyx2lyx (bug 7951)
1246         string const origin = roundtripMode() ? "roundtrip" : outfiledir;
1247         os << "#LyX file created by tex2lyx " << lyx_version_major << '.'
1248            << lyx_version_minor << '\n'
1249            << "\\lyxformat " << LYX_FORMAT << '\n'
1250            << "\\begin_document\n"
1251            << "\\begin_header\n"
1252            << "\\save_transient_properties " << h_save_transient_properties << "\n"
1253            << "\\origin " << origin << "\n"
1254            << "\\textclass " << h_textclass << "\n";
1255         string const raw = subdoc ? empty_string() : h_preamble.str();
1256         if (!raw.empty()) {
1257                 os << "\\begin_preamble\n";
1258                 for (string::size_type i = 0; i < raw.size(); ++i) {
1259                         if (raw[i] == package_beg_sep) {
1260                                 // Here follows some package loading code that
1261                                 // must be skipped if the package is loaded
1262                                 // automatically.
1263                                 string::size_type j = raw.find(package_mid_sep, i);
1264                                 if (j == string::npos)
1265                                         return false;
1266                                 string::size_type k = raw.find(package_end_sep, j);
1267                                 if (k == string::npos)
1268                                         return false;
1269                                 string const package = raw.substr(i + 1, j - i - 1);
1270                                 string const replacement = raw.substr(j + 1, k - j - 1);
1271                                 if (auto_packages.find(package) == auto_packages.end())
1272                                         os << replacement;
1273                                 i = k;
1274                         } else
1275                                 os.put(raw[i]);
1276                 }
1277                 os << "\n\\end_preamble\n";
1278         }
1279         if (!h_options.empty())
1280                 os << "\\options " << h_options << "\n";
1281         os << "\\use_default_options " << h_use_default_options << "\n";
1282         if (!used_modules.empty()) {
1283                 os << "\\begin_modules\n";
1284                 vector<string>::const_iterator const end = used_modules.end();
1285                 vector<string>::const_iterator it = used_modules.begin();
1286                 for (; it != end; ++it)
1287                         os << *it << '\n';
1288                 os << "\\end_modules\n";
1289         }
1290         if (!h_includeonlys.empty()) {
1291                 os << "\\begin_includeonly\n";
1292                 for (auto const & iofile : h_includeonlys)
1293                         os << iofile << '\n';
1294                 os << "\\end_includeonly\n";
1295         }
1296         os << "\\maintain_unincluded_children " << h_maintain_unincluded_children << "\n"
1297            << "\\language " << h_language << "\n"
1298            << "\\language_package " << h_language_package << "\n"
1299            << "\\inputencoding " << h_inputencoding << "\n"
1300            << "\\fontencoding " << h_fontencoding << "\n"
1301            << "\\font_roman \"" << h_font_roman[0]
1302            << "\" \"" << h_font_roman[1] << "\"\n"
1303            << "\\font_sans \"" << h_font_sans[0] << "\" \"" << h_font_sans[1] << "\"\n"
1304            << "\\font_typewriter \"" << h_font_typewriter[0]
1305            << "\" \"" << h_font_typewriter[1] << "\"\n"
1306            << "\\font_math \"" << h_font_math[0] << "\" \"" << h_font_math[1] << "\"\n"
1307            << "\\font_default_family " << h_font_default_family << "\n"
1308            << "\\use_non_tex_fonts " << (h_use_non_tex_fonts ? "true" : "false") << '\n'
1309            << "\\font_sc " << h_font_sc << "\n"
1310            << "\\font_osf " << h_font_osf << "\n"
1311            << "\\font_sf_scale " << h_font_sf_scale[0]
1312            << ' ' << h_font_sf_scale[1] << '\n'
1313            << "\\font_tt_scale " << h_font_tt_scale[0]
1314            << ' ' << h_font_tt_scale[1] << '\n';
1315         if (!h_font_cjk.empty())
1316                 os << "\\font_cjk " << h_font_cjk << '\n';
1317         os << "\\use_microtype " << h_use_microtype << '\n'
1318            << "\\use_dash_ligatures " << h_use_dash_ligatures << '\n'
1319            << "\\graphics " << h_graphics << '\n'
1320            << "\\default_output_format " << h_default_output_format << "\n"
1321            << "\\output_sync " << h_output_sync << "\n";
1322         if (h_output_sync == "1")
1323                 os << "\\output_sync_macro \"" << h_output_sync_macro << "\"\n";
1324         os << "\\bibtex_command " << h_bibtex_command << "\n"
1325            << "\\index_command " << h_index_command << "\n";
1326         if (!h_float_placement.empty())
1327                 os << "\\float_placement " << h_float_placement << "\n";
1328         os << "\\paperfontsize " << h_paperfontsize << "\n"
1329            << "\\spacing " << h_spacing << "\n"
1330            << "\\use_hyperref " << h_use_hyperref << '\n';
1331         if (h_use_hyperref == "true") {
1332                 if (!h_pdf_title.empty())
1333                         os << "\\pdf_title " << Lexer::quoteString(h_pdf_title) << '\n';
1334                 if (!h_pdf_author.empty())
1335                         os << "\\pdf_author " << Lexer::quoteString(h_pdf_author) << '\n';
1336                 if (!h_pdf_subject.empty())
1337                         os << "\\pdf_subject " << Lexer::quoteString(h_pdf_subject) << '\n';
1338                 if (!h_pdf_keywords.empty())
1339                         os << "\\pdf_keywords " << Lexer::quoteString(h_pdf_keywords) << '\n';
1340                 os << "\\pdf_bookmarks " << h_pdf_bookmarks << "\n"
1341                       "\\pdf_bookmarksnumbered " << h_pdf_bookmarksnumbered << "\n"
1342                       "\\pdf_bookmarksopen " << h_pdf_bookmarksopen << "\n"
1343                       "\\pdf_bookmarksopenlevel " << h_pdf_bookmarksopenlevel << "\n"
1344                       "\\pdf_breaklinks " << h_pdf_breaklinks << "\n"
1345                       "\\pdf_pdfborder " << h_pdf_pdfborder << "\n"
1346                       "\\pdf_colorlinks " << h_pdf_colorlinks << "\n"
1347                       "\\pdf_backref " << h_pdf_backref << "\n"
1348                       "\\pdf_pdfusetitle " << h_pdf_pdfusetitle << '\n';
1349                 if (!h_pdf_pagemode.empty())
1350                         os << "\\pdf_pagemode " << h_pdf_pagemode << '\n';
1351                 if (!h_pdf_quoted_options.empty())
1352                         os << "\\pdf_quoted_options " << Lexer::quoteString(h_pdf_quoted_options) << '\n';
1353         }
1354         os << "\\papersize " << h_papersize << "\n"
1355            << "\\use_geometry " << h_use_geometry << '\n';
1356         for (map<string, string>::const_iterator it = h_use_packages.begin();
1357              it != h_use_packages.end(); ++it)
1358                 os << "\\use_package " << it->first << ' ' << it->second << '\n';
1359         os << "\\cite_engine " << h_cite_engine << '\n'
1360            << "\\cite_engine_type " << h_cite_engine_type << '\n'
1361            << "\\biblio_style " << h_biblio_style << "\n"
1362            << "\\use_bibtopic " << h_use_bibtopic << "\n";
1363         if (!h_biblio_options.empty())
1364                 os << "\\biblio_options " << h_biblio_options << "\n";
1365         if (!h_biblatex_bibstyle.empty())
1366                 os << "\\biblatex_bibstyle " << h_biblatex_bibstyle << "\n";
1367         if (!h_biblatex_citestyle.empty())
1368                 os << "\\biblatex_citestyle " << h_biblatex_citestyle << "\n";
1369         if (!h_multibib.empty())
1370                 os << "\\multibib " << h_multibib << "\n";
1371         os << "\\use_indices " << h_use_indices << "\n"
1372            << "\\paperorientation " << h_paperorientation << '\n'
1373            << "\\suppress_date " << h_suppress_date << '\n'
1374            << "\\justification " << h_justification << '\n'
1375            << "\\use_refstyle " << h_use_refstyle << '\n'
1376            << "\\use_minted " << h_use_minted << '\n';
1377         if (!h_fontcolor.empty())
1378                 os << "\\fontcolor " << h_fontcolor << '\n';
1379         if (!h_notefontcolor.empty())
1380                 os << "\\notefontcolor " << h_notefontcolor << '\n';
1381         if (!h_backgroundcolor.empty())
1382                 os << "\\backgroundcolor " << h_backgroundcolor << '\n';
1383         if (!h_boxbgcolor.empty())
1384                 os << "\\boxbgcolor " << h_boxbgcolor << '\n';
1385         if (index_number != 0)
1386                 for (int i = 0; i < index_number; i++) {
1387                         os << "\\index " << h_index[i] << '\n'
1388                            << "\\shortcut " << h_shortcut[i] << '\n'
1389                            << "\\color " << h_color << '\n'
1390                            << "\\end_index\n";
1391                 }
1392         else {
1393                 os << "\\index " << h_index[0] << '\n'
1394                    << "\\shortcut " << h_shortcut[0] << '\n'
1395                    << "\\color " << h_color << '\n'
1396                    << "\\end_index\n";
1397         }
1398         os << h_margins
1399            << "\\secnumdepth " << h_secnumdepth << "\n"
1400            << "\\tocdepth " << h_tocdepth << "\n"
1401            << "\\paragraph_separation " << h_paragraph_separation << "\n";
1402         if (h_paragraph_separation == "skip")
1403                 os << "\\defskip " << h_defskip << "\n";
1404         else
1405                 os << "\\paragraph_indentation " << h_paragraph_indentation << "\n";
1406         os << "\\is_math_indent " << h_is_mathindent << "\n";
1407         if (!h_mathindentation.empty())
1408                 os << "\\math_indentation " << h_mathindentation << "\n";
1409         os << "\\math_numbering_side " << h_math_numbering_side << "\n";
1410         os << "\\quotes_style " << h_quotes_style << "\n"
1411            << "\\dynamic_quotes " << h_dynamic_quotes << "\n"
1412            << "\\papercolumns " << h_papercolumns << "\n"
1413            << "\\papersides " << h_papersides << "\n"
1414            << "\\paperpagestyle " << h_paperpagestyle << "\n";
1415         if (!h_listings_params.empty())
1416                 os << "\\listings_params " << h_listings_params << "\n";
1417         os << "\\tracking_changes " << h_tracking_changes << "\n"
1418            << "\\output_changes " << h_output_changes << "\n"
1419            << "\\html_math_output " << h_html_math_output << "\n"
1420            << "\\html_css_as_file " << h_html_css_as_file << "\n"
1421            << "\\html_be_strict " << h_html_be_strict << "\n"
1422            << authors_
1423            << "\\end_header\n\n"
1424            << "\\begin_body\n";
1425         return true;
1426 }
1427
1428
1429 void Preamble::parse(Parser & p, string const & forceclass,
1430                      TeX2LyXDocClass & tc)
1431 {
1432         // initialize fixed types
1433         special_columns_['D'] = 3;
1434         parse(p, forceclass, false, tc);
1435 }
1436
1437
1438 void Preamble::parse(Parser & p, string const & forceclass,
1439                      bool detectEncoding, TeX2LyXDocClass & tc)
1440 {
1441         bool is_full_document = false;
1442         bool is_lyx_file = false;
1443         bool in_lyx_preamble = false;
1444
1445         // determine whether this is a full document or a fragment for inclusion
1446         while (p.good()) {
1447                 Token const & t = p.get_token();
1448
1449                 if (t.cat() == catEscape && t.cs() == "documentclass") {
1450                         is_full_document = true;
1451                         break;
1452                 }
1453         }
1454         p.reset();
1455
1456         if (detectEncoding && !is_full_document)
1457                 return;
1458
1459         while (is_full_document && p.good()) {
1460                 if (detectEncoding && h_inputencoding != "auto" &&
1461                     h_inputencoding != "default")
1462                         return;
1463
1464                 Token const & t = p.get_token();
1465
1466 #ifdef FILEDEBUG
1467                 if (!detectEncoding)
1468                         cerr << "t: " << t << '\n';
1469 #endif
1470
1471                 //
1472                 // cat codes
1473                 //
1474                 if (!in_lyx_preamble &&
1475                     (t.cat() == catLetter ||
1476                      t.cat() == catSuper ||
1477                      t.cat() == catSub ||
1478                      t.cat() == catOther ||
1479                      t.cat() == catMath ||
1480                      t.cat() == catActive ||
1481                      t.cat() == catBegin ||
1482                      t.cat() == catEnd ||
1483                      t.cat() == catAlign ||
1484                      t.cat() == catParameter)) {
1485                         h_preamble << t.cs();
1486                         continue;
1487                 }
1488
1489                 if (!in_lyx_preamble &&
1490                     (t.cat() == catSpace || t.cat() == catNewline)) {
1491                         h_preamble << t.asInput();
1492                         continue;
1493                 }
1494
1495                 if (t.cat() == catComment) {
1496                         static regex const islyxfile("%% LyX .* created this file");
1497                         static regex const usercommands("User specified LaTeX commands");
1498
1499                         string const comment = t.asInput();
1500
1501                         // magically switch encoding default if it looks like XeLaTeX
1502                         static string const magicXeLaTeX =
1503                                 "% This document must be compiled with XeLaTeX ";
1504                         if (comment.size() > magicXeLaTeX.size()
1505                                   && comment.substr(0, magicXeLaTeX.size()) == magicXeLaTeX
1506                                   && h_inputencoding == "auto") {
1507                                 if (!detectEncoding)
1508                                         cerr << "XeLaTeX comment found, switching to UTF8\n";
1509                                 h_inputencoding = "utf8";
1510                         }
1511                         smatch sub;
1512                         if (regex_search(comment, sub, islyxfile)) {
1513                                 is_lyx_file = true;
1514                                 in_lyx_preamble = true;
1515                         } else if (is_lyx_file
1516                                    && regex_search(comment, sub, usercommands))
1517                                 in_lyx_preamble = false;
1518                         else if (!in_lyx_preamble)
1519                                 h_preamble << t.asInput();
1520                         continue;
1521                 }
1522
1523                 if (t.cs() == "PassOptionsToPackage") {
1524                         string const poptions = p.getArg('{', '}');
1525                         string const package = p.verbatim_item();
1526                         extra_package_options_.insert(make_pair(package, poptions));
1527                         continue;
1528                 }
1529
1530                 if (t.cs() == "pagestyle") {
1531                         h_paperpagestyle = p.verbatim_item();
1532                         continue;
1533                 }
1534
1535                 if (t.cs() == "setdefaultlanguage") {
1536                         xetex = true;
1537                         // We don't yet care about non-language variant options
1538                         // because LyX doesn't support this yet, see bug #8214
1539                         if (p.hasOpt()) {
1540                                 string langopts = p.getOpt();
1541                                 // check if the option contains a variant, if yes, extract it
1542                                 string::size_type pos_var = langopts.find("variant");
1543                                 string::size_type i = langopts.find(',', pos_var);
1544                                 string::size_type k = langopts.find('=', pos_var);
1545                                 if (pos_var != string::npos){
1546                                         string variant;
1547                                         if (i == string::npos)
1548                                                 variant = langopts.substr(k + 1, langopts.length() - k - 2);
1549                                         else
1550                                                 variant = langopts.substr(k + 1, i - k - 1);
1551                                         h_language = variant;
1552                                 }
1553                                 p.verbatim_item();
1554                         } else
1555                                 h_language = p.verbatim_item();
1556                         //finally translate the poyglossia name to a LyX name
1557                         h_language = polyglossia2lyx(h_language);
1558                         continue;
1559                 }
1560
1561                 if (t.cs() == "setotherlanguage") {
1562                         // We don't yet care about the option because LyX doesn't
1563                         // support this yet, see bug #8214
1564                         p.hasOpt() ? p.getOpt() : string();
1565                         p.verbatim_item();
1566                         continue;
1567                 }
1568
1569                 if (t.cs() == "setmainfont") {
1570                         // we don't care about the option
1571                         p.hasOpt() ? p.getOpt() : string();
1572                         h_font_roman[1] = p.getArg('{', '}');
1573                         continue;
1574                 }
1575
1576                 if (t.cs() == "setsansfont" || t.cs() == "setmonofont") {
1577                         // LyX currently only supports the scale option
1578                         string scale;
1579                         if (p.hasOpt()) {
1580                                 string fontopts = p.getArg('[', ']');
1581                                 // check if the option contains a scaling, if yes, extract it
1582                                 string::size_type pos = fontopts.find("Scale");
1583                                 if (pos != string::npos) {
1584                                         string::size_type i = fontopts.find(',', pos);
1585                                         if (i == string::npos)
1586                                                 scale_as_percentage(fontopts.substr(pos + 1), scale);
1587                                         else
1588                                                 scale_as_percentage(fontopts.substr(pos, i - pos), scale);
1589                                 }
1590                         }
1591                         if (t.cs() == "setsansfont") {
1592                                 if (!scale.empty())
1593                                         h_font_sf_scale[1] = scale;
1594                                 h_font_sans[1] = p.getArg('{', '}');
1595                         } else {
1596                                 if (!scale.empty())
1597                                         h_font_tt_scale[1] = scale;
1598                                 h_font_typewriter[1] = p.getArg('{', '}');
1599                         }
1600                         continue;
1601                 }
1602
1603                 else if (t.cs() == "date") {
1604                         string argument = p.getArg('{', '}');
1605                         if (argument.empty())
1606                                 h_suppress_date = "true";
1607                         else
1608                                 h_preamble << t.asInput() << '{' << argument << '}';
1609                 }
1610
1611                 if (t.cs() == "color") {
1612                         string const space =
1613                                 (p.hasOpt() ? p.getOpt() : string());
1614                         string argument = p.getArg('{', '}');
1615                         // check the case that a standard color is used
1616                         if (space.empty() && is_known(argument, known_basic_colors)) {
1617                                 h_fontcolor = rgbcolor2code(argument);
1618                                 registerAutomaticallyLoadedPackage("color");
1619                         } else if (space.empty() && argument == "document_fontcolor")
1620                                 registerAutomaticallyLoadedPackage("color");
1621                         // check the case that LyX's document_fontcolor is defined
1622                         // but not used for \color
1623                         else {
1624                                 h_preamble << t.asInput();
1625                                 if (!space.empty())
1626                                         h_preamble << space;
1627                                 h_preamble << '{' << argument << '}';
1628                                 // the color might already be set because \definecolor
1629                                 // is parsed before this
1630                                 h_fontcolor = "";
1631                         }
1632                         continue;
1633                 }
1634
1635                 if (t.cs() == "pagecolor") {
1636                         string argument = p.getArg('{', '}');
1637                         // check the case that a standard color is used
1638                         if (is_known(argument, known_basic_colors)) {
1639                                 h_backgroundcolor = rgbcolor2code(argument);
1640                         } else if (argument == "page_backgroundcolor")
1641                                 registerAutomaticallyLoadedPackage("color");
1642                         // check the case that LyX's page_backgroundcolor is defined
1643                         // but not used for \pagecolor
1644                         else {
1645                                 h_preamble << t.asInput() << '{' << argument << '}';
1646                                 // the color might already be set because \definecolor
1647                                 // is parsed before this
1648                                 h_backgroundcolor = "";
1649                         }
1650                         continue;
1651                 }
1652
1653                 if (t.cs() == "makeatletter") {
1654                         // LyX takes care of this
1655                         p.setCatcode('@', catLetter);
1656                         continue;
1657                 }
1658
1659                 if (t.cs() == "makeatother") {
1660                         // LyX takes care of this
1661                         p.setCatcode('@', catOther);
1662                         continue;
1663                 }
1664
1665                 if (t.cs() == "makeindex") {
1666                         // LyX will re-add this if a print index command is found
1667                         p.skip_spaces();
1668                         continue;
1669                 }
1670
1671                 if (t.cs() == "newindex") {
1672                         string const indexname = p.getArg('[', ']');
1673                         string const shortcut = p.verbatim_item();
1674                         if (!indexname.empty())
1675                                 h_index[index_number] = indexname;
1676                         else
1677                                 h_index[index_number] = shortcut;
1678                         h_shortcut[index_number] = shortcut;
1679                         index_number += 1;
1680                         p.skip_spaces();
1681                         continue;
1682                 }
1683
1684                 if (t.cs() == "addbibresource") {
1685                         biblatex_bibliographies.push_back(removeExtension(p.getArg('{', '}')));
1686                         continue;
1687                 }
1688
1689                 if (t.cs() == "bibliography") {
1690                         vector<string> bibs = getVectorFromString(p.getArg('{', '}'));
1691                         biblatex_bibliographies.insert(biblatex_bibliographies.end(), bibs.begin(), bibs.end());
1692                         continue;
1693                 }
1694
1695                 if (t.cs() == "RS@ifundefined") {
1696                         string const name = p.verbatim_item();
1697                         string const body1 = p.verbatim_item();
1698                         string const body2 = p.verbatim_item();
1699                         // only non-lyxspecific stuff
1700                         if (in_lyx_preamble &&
1701                             (name == "subsecref" || name == "thmref" || name == "lemref"))
1702                                 p.skip_spaces();
1703                         else {
1704                                 ostringstream ss;
1705                                 ss << '\\' << t.cs();
1706                                 ss << '{' << name << '}'
1707                                    << '{' << body1 << '}'
1708                                    << '{' << body2 << '}';
1709                                 h_preamble << ss.str();
1710                         }
1711                         continue;
1712                 }
1713
1714                 if (t.cs() == "AtBeginDocument") {
1715                         string const name = p.verbatim_item();
1716                         // only non-lyxspecific stuff
1717                         if (in_lyx_preamble &&
1718                             (name == "\\providecommand\\partref[1]{\\ref{part:#1}}"
1719                                 || name == "\\providecommand\\chapref[1]{\\ref{chap:#1}}"
1720                                 || name == "\\providecommand\\secref[1]{\\ref{sec:#1}}"
1721                                 || name == "\\providecommand\\subsecref[1]{\\ref{subsec:#1}}"
1722                                 || name == "\\providecommand\\parref[1]{\\ref{par:#1}}"
1723                                 || name == "\\providecommand\\figref[1]{\\ref{fig:#1}}"
1724                                 || name == "\\providecommand\\tabref[1]{\\ref{tab:#1}}"
1725                                 || name == "\\providecommand\\algref[1]{\\ref{alg:#1}}"
1726                                 || name == "\\providecommand\\fnref[1]{\\ref{fn:#1}}"
1727                                 || name == "\\providecommand\\enuref[1]{\\ref{enu:#1}}"
1728                                 || name == "\\providecommand\\eqref[1]{\\ref{eq:#1}}"
1729                                 || name == "\\providecommand\\lemref[1]{\\ref{lem:#1}}"
1730                                 || name == "\\providecommand\\thmref[1]{\\ref{thm:#1}}"
1731                                 || name == "\\providecommand\\corref[1]{\\ref{cor:#1}}"
1732                                 || name == "\\providecommand\\propref[1]{\\ref{prop:#1}}"))
1733                                 p.skip_spaces();
1734                         else {
1735                                 ostringstream ss;
1736                                 ss << '\\' << t.cs();
1737                                 ss << '{' << name << '}';
1738                                 h_preamble << ss.str();
1739                         }
1740                         continue;
1741                 }
1742
1743                 if (t.cs() == "newcommand" || t.cs() == "newcommandx"
1744                     || t.cs() == "renewcommand" || t.cs() == "renewcommandx"
1745                     || t.cs() == "providecommand" || t.cs() == "providecommandx"
1746                     || t.cs() == "DeclareRobustCommand"
1747                     || t.cs() == "DeclareRobustCommandx"
1748                     || t.cs() == "ProvideTextCommandDefault"
1749                     || t.cs() == "DeclareMathAccent") {
1750                         bool star = false;
1751                         if (p.next_token().character() == '*') {
1752                                 p.get_token();
1753                                 star = true;
1754                         }
1755                         string const name = p.verbatim_item();
1756                         string const opt1 = p.getFullOpt();
1757                         string const opt2 = p.getFullOpt();
1758                         string const body = p.verbatim_item();
1759                         // store the in_lyx_preamble setting
1760                         bool const was_in_lyx_preamble = in_lyx_preamble;
1761                         // font settings
1762                         if (name == "\\rmdefault")
1763                                 if (is_known(body, known_roman_fonts)) {
1764                                         h_font_roman[0] = body;
1765                                         p.skip_spaces();
1766                                         in_lyx_preamble = true;
1767                                 }
1768                         if (name == "\\sfdefault")
1769                                 if (is_known(body, known_sans_fonts)) {
1770                                         h_font_sans[0] = body;
1771                                         p.skip_spaces();
1772                                         in_lyx_preamble = true;
1773                                 }
1774                         if (name == "\\ttdefault")
1775                                 if (is_known(body, known_typewriter_fonts)) {
1776                                         h_font_typewriter[0] = body;
1777                                         p.skip_spaces();
1778                                         in_lyx_preamble = true;
1779                                 }
1780                         if (name == "\\familydefault") {
1781                                 string family = body;
1782                                 // remove leading "\"
1783                                 h_font_default_family = family.erase(0,1);
1784                                 p.skip_spaces();
1785                                 in_lyx_preamble = true;
1786                         }
1787
1788                         // remove LyX-specific definitions that are re-added by LyX
1789                         // if necessary
1790                         // \lyxline is an ancient command that is converted by tex2lyx into
1791                         // a \rule therefore remove its preamble code
1792                         if (name == "\\lyxdot" || name == "\\lyxarrow"
1793                             || name == "\\lyxline" || name == "\\LyX") {
1794                                 p.skip_spaces();
1795                                 in_lyx_preamble = true;
1796                         }
1797
1798                         // Add the command to the known commands
1799                         add_known_command(name, opt1, !opt2.empty(), from_utf8(body));
1800
1801                         // only non-lyxspecific stuff
1802                         if (!in_lyx_preamble) {
1803                                 ostringstream ss;
1804                                 ss << '\\' << t.cs();
1805                                 if (star)
1806                                         ss << '*';
1807                                 ss << '{' << name << '}' << opt1 << opt2
1808                                    << '{' << body << "}";
1809                                 h_preamble << ss.str();
1810 /*
1811                                 ostream & out = in_preamble ? h_preamble : os;
1812                                 out << "\\" << t.cs() << "{" << name << "}"
1813                                     << opts << "{" << body << "}";
1814 */
1815                         }
1816                         // restore the in_lyx_preamble setting
1817                         in_lyx_preamble = was_in_lyx_preamble;
1818                         continue;
1819                 }
1820
1821                 if (t.cs() == "documentclass") {
1822                         vector<string>::iterator it;
1823                         vector<string> opts = split_options(p.getArg('[', ']'));
1824                         handle_opt(opts, known_fontsizes, h_paperfontsize);
1825                         delete_opt(opts, known_fontsizes);
1826                         // delete "pt" at the end
1827                         string::size_type i = h_paperfontsize.find("pt");
1828                         if (i != string::npos)
1829                                 h_paperfontsize.erase(i);
1830                         // The documentclass options are always parsed before the options
1831                         // of the babel call so that a language cannot overwrite the babel
1832                         // options.
1833                         handle_opt(opts, known_languages, h_language);
1834                         delete_opt(opts, known_languages);
1835
1836                         // math indentation
1837                         if ((it = find(opts.begin(), opts.end(), "fleqn"))
1838                                  != opts.end()) {
1839                                 h_is_mathindent = "1";
1840                                 opts.erase(it);
1841                         }
1842                         // formula numbering side
1843                         if ((it = find(opts.begin(), opts.end(), "leqno"))
1844                                  != opts.end()) {
1845                                 h_math_numbering_side = "left";
1846                                 opts.erase(it);
1847                         }
1848                         else if ((it = find(opts.begin(), opts.end(), "reqno"))
1849                                  != opts.end()) {
1850                                 h_math_numbering_side = "right";
1851                                 opts.erase(it);
1852                         }
1853
1854                         // paper orientation
1855                         if ((it = find(opts.begin(), opts.end(), "landscape")) != opts.end()) {
1856                                 h_paperorientation = "landscape";
1857                                 opts.erase(it);
1858                         }
1859                         // paper sides
1860                         if ((it = find(opts.begin(), opts.end(), "oneside"))
1861                                  != opts.end()) {
1862                                 h_papersides = "1";
1863                                 opts.erase(it);
1864                         }
1865                         if ((it = find(opts.begin(), opts.end(), "twoside"))
1866                                  != opts.end()) {
1867                                 h_papersides = "2";
1868                                 opts.erase(it);
1869                         }
1870                         // paper columns
1871                         if ((it = find(opts.begin(), opts.end(), "onecolumn"))
1872                                  != opts.end()) {
1873                                 h_papercolumns = "1";
1874                                 opts.erase(it);
1875                         }
1876                         if ((it = find(opts.begin(), opts.end(), "twocolumn"))
1877                                  != opts.end()) {
1878                                 h_papercolumns = "2";
1879                                 opts.erase(it);
1880                         }
1881                         // paper sizes
1882                         // some size options are known to any document classes, other sizes
1883                         // are handled by the \geometry command of the geometry package
1884                         handle_opt(opts, known_class_paper_sizes, h_papersize);
1885                         delete_opt(opts, known_class_paper_sizes);
1886                         // the remaining options
1887                         h_options = join(opts, ",");
1888                         // FIXME This does not work for classes that have a
1889                         //       different name in LyX than in LaTeX
1890                         h_textclass = p.getArg('{', '}');
1891                         p.skip_spaces();
1892                         continue;
1893                 }
1894
1895                 if (t.cs() == "usepackage") {
1896                         string const options = p.getArg('[', ']');
1897                         string const name = p.getArg('{', '}');
1898                         vector<string> vecnames;
1899                         split(name, vecnames, ',');
1900                         vector<string>::const_iterator it  = vecnames.begin();
1901                         vector<string>::const_iterator end = vecnames.end();
1902                         for (; it != end; ++it)
1903                                 handle_package(p, trimSpaceAndEol(*it), options,
1904                                                in_lyx_preamble, detectEncoding);
1905                         continue;
1906                 }
1907
1908                 if (t.cs() == "inputencoding") {
1909                         string const encoding = p.getArg('{','}');
1910                         Encoding const * const enc = encodings.fromLaTeXName(
1911                                 encoding, Encoding::inputenc, true);
1912                         if (!enc) {
1913                                 if (!detectEncoding)
1914                                         cerr << "Unknown encoding " << encoding
1915                                              << ". Ignoring." << std::endl;
1916                         } else {
1917                                 if (!enc->unsafe())
1918                                         h_inputencoding = enc->name();
1919                                 p.setEncoding(enc->iconvName());
1920                         }
1921                         continue;
1922                 }
1923
1924                 if (t.cs() == "newenvironment") {
1925                         string const name = p.getArg('{', '}');
1926                         string const opt1 = p.getFullOpt();
1927                         string const opt2 = p.getFullOpt();
1928                         string const beg = p.verbatim_item();
1929                         string const end = p.verbatim_item();
1930                         if (!in_lyx_preamble) {
1931                                 h_preamble << "\\newenvironment{" << name
1932                                            << '}' << opt1 << opt2 << '{'
1933                                            << beg << "}{" << end << '}';
1934                         }
1935                         add_known_environment(name, opt1, !opt2.empty(),
1936                                               from_utf8(beg), from_utf8(end));
1937                         continue;
1938                 }
1939
1940                 if (t.cs() == "newtheorem") {
1941                         bool star = false;
1942                         if (p.next_token().character() == '*') {
1943                                 p.get_token();
1944                                 star = true;
1945                         }
1946                         string const name = p.getArg('{', '}');
1947                         string const opt1 = p.getFullOpt();
1948                         string const opt2 = p.getFullOpt();
1949                         string const body = p.verbatim_item();
1950                         string const opt3 = p.getFullOpt();
1951                         string const cmd = star ? "\\newtheorem*" : "\\newtheorem";
1952
1953                         string const complete = cmd + "{" + name + '}' +
1954                                           opt1 + opt2 + '{' + body + '}' + opt3;
1955
1956                         add_known_theorem(name, opt1, !opt2.empty(), from_utf8(complete));
1957
1958                         if (!in_lyx_preamble)
1959                                 h_preamble << complete;
1960                         continue;
1961                 }
1962
1963                 if (t.cs() == "def") {
1964                         string name = p.get_token().cs();
1965                         // In fact, name may be more than the name:
1966                         // In the test case of bug 8116
1967                         // name == "csname SF@gobble@opt \endcsname".
1968                         // Therefore, we need to use asInput() instead of cs().
1969                         while (p.next_token().cat() != catBegin)
1970                                 name += p.get_token().asInput();
1971                         if (!in_lyx_preamble)
1972                                 h_preamble << "\\def\\" << name << '{'
1973                                            << p.verbatim_item() << "}";
1974                         continue;
1975                 }
1976
1977                 if (t.cs() == "newcolumntype") {
1978                         string const name = p.getArg('{', '}');
1979                         trimSpaceAndEol(name);
1980                         int nargs = 0;
1981                         string opts = p.getOpt();
1982                         if (!opts.empty()) {
1983                                 istringstream is(string(opts, 1));
1984                                 is >> nargs;
1985                         }
1986                         special_columns_[name[0]] = nargs;
1987                         h_preamble << "\\newcolumntype{" << name << "}";
1988                         if (nargs)
1989                                 h_preamble << "[" << nargs << "]";
1990                         h_preamble << "{" << p.verbatim_item() << "}";
1991                         continue;
1992                 }
1993
1994                 if (t.cs() == "setcounter") {
1995                         string const name = p.getArg('{', '}');
1996                         string const content = p.getArg('{', '}');
1997                         if (name == "secnumdepth")
1998                                 h_secnumdepth = content;
1999                         else if (name == "tocdepth")
2000                                 h_tocdepth = content;
2001                         else
2002                                 h_preamble << "\\setcounter{" << name << "}{" << content << "}";
2003                         continue;
2004                 }
2005
2006                 if (t.cs() == "setlength") {
2007                         string const name = p.verbatim_item();
2008                         string const content = p.verbatim_item();
2009                         // the paragraphs are only not indented when \parindent is set to zero
2010                         if (name == "\\parindent" && content != "") {
2011                                 if (content[0] == '0')
2012                                         h_paragraph_separation = "skip";
2013                                 else
2014                                         h_paragraph_indentation = translate_len(content);
2015                         } else if (name == "\\parskip") {
2016                                 if (content == "\\smallskipamount")
2017                                         h_defskip = "smallskip";
2018                                 else if (content == "\\medskipamount")
2019                                         h_defskip = "medskip";
2020                                 else if (content == "\\bigskipamount")
2021                                         h_defskip = "bigskip";
2022                                 else
2023                                         h_defskip = translate_len(content);
2024                         } else if (name == "\\mathindent") {
2025                                 h_mathindentation = translate_len(content);
2026                         } else
2027                                 h_preamble << "\\setlength{" << name << "}{" << content << "}";
2028                         continue;
2029                 }
2030
2031                 if (t.cs() == "onehalfspacing") {
2032                         h_spacing = "onehalf";
2033                         continue;
2034                 }
2035
2036                 if (t.cs() == "doublespacing") {
2037                         h_spacing = "double";
2038                         continue;
2039                 }
2040
2041                 if (t.cs() == "setstretch") {
2042                         h_spacing = "other " + p.verbatim_item();
2043                         continue;
2044                 }
2045
2046                 if (t.cs() == "synctex") {
2047                         // the scheme is \synctex=value
2048                         // where value can only be "1" or "-1"
2049                         h_output_sync = "1";
2050                         // there can be any character behind the value (e.g. a linebreak or a '\'
2051                         // therefore we extract it char by char
2052                         p.get_token();
2053                         string value = p.get_token().asInput();
2054                         if (value == "-")
2055                                 value += p.get_token().asInput();
2056                         h_output_sync_macro = "\\synctex=" + value;
2057                         continue;
2058                 }
2059
2060                 if (t.cs() == "begin") {
2061                         string const name = p.getArg('{', '}');
2062                         if (name == "document")
2063                                 break;
2064                         h_preamble << "\\begin{" << name << "}";
2065                         continue;
2066                 }
2067
2068                 if (t.cs() == "geometry") {
2069                         vector<string> opts = split_options(p.getArg('{', '}'));
2070                         handle_geometry(opts);
2071                         continue;
2072                 }
2073
2074                 if (t.cs() == "definecolor") {
2075                         string const color = p.getArg('{', '}');
2076                         string const space = p.getArg('{', '}');
2077                         string const value = p.getArg('{', '}');
2078                         if (color == "document_fontcolor" && space == "rgb") {
2079                                 RGBColor c(RGBColorFromLaTeX(value));
2080                                 h_fontcolor = X11hexname(c);
2081                         } else if (color == "note_fontcolor" && space == "rgb") {
2082                                 RGBColor c(RGBColorFromLaTeX(value));
2083                                 h_notefontcolor = X11hexname(c);
2084                         } else if (color == "page_backgroundcolor" && space == "rgb") {
2085                                 RGBColor c(RGBColorFromLaTeX(value));
2086                                 h_backgroundcolor = X11hexname(c);
2087                         } else if (color == "shadecolor" && space == "rgb") {
2088                                 RGBColor c(RGBColorFromLaTeX(value));
2089                                 h_boxbgcolor = X11hexname(c);
2090                         } else {
2091                                 h_preamble << "\\definecolor{" << color
2092                                            << "}{" << space << "}{" << value
2093                                            << '}';
2094                         }
2095                         continue;
2096                 }
2097
2098                 if (t.cs() == "bibliographystyle") {
2099                         h_biblio_style = p.verbatim_item();
2100                         continue;
2101                 }
2102
2103                 if (t.cs() == "jurabibsetup") {
2104                         // FIXME p.getArg('{', '}') is most probably wrong (it
2105                         //       does not handle nested braces).
2106                         //       Use p.verbatim_item() instead.
2107                         vector<string> jurabibsetup =
2108                                 split_options(p.getArg('{', '}'));
2109                         // add jurabibsetup to the jurabib package options
2110                         add_package("jurabib", jurabibsetup);
2111                         if (!jurabibsetup.empty()) {
2112                                 h_preamble << "\\jurabibsetup{"
2113                                            << join(jurabibsetup, ",") << '}';
2114                         }
2115                         continue;
2116                 }
2117
2118                 if (t.cs() == "hypersetup") {
2119                         vector<string> hypersetup =
2120                                 split_options(p.verbatim_item());
2121                         // add hypersetup to the hyperref package options
2122                         handle_hyperref(hypersetup);
2123                         if (!hypersetup.empty()) {
2124                                 h_preamble << "\\hypersetup{"
2125                                            << join(hypersetup, ",") << '}';
2126                         }
2127                         continue;
2128                 }
2129
2130                 if (t.cs() == "includeonly") {
2131                         vector<string> includeonlys = getVectorFromString(p.getArg('{', '}'));
2132                         for (auto & iofile : includeonlys) {
2133                                 string filename(normalize_filename(iofile));
2134                                 string const path = getMasterFilePath(true);
2135                                 // We want to preserve relative/absolute filenames,
2136                                 // therefore path is only used for testing
2137                                 if (!makeAbsPath(filename, path).exists()) {
2138                                         // The file extension is probably missing.
2139                                         // Now try to find it out.
2140                                         string const tex_name =
2141                                                 find_file(filename, path,
2142                                                           known_tex_extensions);
2143                                         if (!tex_name.empty())
2144                                                 filename = tex_name;
2145                                 }
2146                                 string outname;
2147                                 if (makeAbsPath(filename, path).exists())
2148                                         fix_child_filename(filename);
2149                                 else
2150                                         cerr << "Warning: Could not find included file '"
2151                                              << filename << "'." << endl;
2152                                 outname = changeExtension(filename, "lyx");
2153                                 h_includeonlys.push_back(outname);
2154                         }
2155                         continue;
2156                 }
2157
2158                 if (is_known(t.cs(), known_if_3arg_commands)) {
2159                         // prevent misparsing of \usepackage if it is used
2160                         // as an argument (see e.g. our own output of
2161                         // \@ifundefined above)
2162                         string const arg1 = p.verbatim_item();
2163                         string const arg2 = p.verbatim_item();
2164                         string const arg3 = p.verbatim_item();
2165                         // test case \@ifundefined{date}{}{\date{}}
2166                         if (t.cs() == "@ifundefined" && arg1 == "date" &&
2167                             arg2.empty() && arg3 == "\\date{}") {
2168                                 h_suppress_date = "true";
2169                         // older tex2lyx versions did output
2170                         // \@ifundefined{definecolor}{\usepackage{color}}{}
2171                         } else if (t.cs() == "@ifundefined" &&
2172                                    arg1 == "definecolor" &&
2173                                    arg2 == "\\usepackage{color}" &&
2174                                    arg3.empty()) {
2175                                 if (!in_lyx_preamble)
2176                                         h_preamble << package_beg_sep
2177                                                    << "color"
2178                                                    << package_mid_sep
2179                                                    << "\\@ifundefined{definecolor}{color}{}"
2180                                                    << package_end_sep;
2181                         // test for case
2182                         //\@ifundefined{showcaptionsetup}{}{%
2183                         // \PassOptionsToPackage{caption=false}{subfig}}
2184                         // that LyX uses for subfloats
2185                         } else if (t.cs() == "@ifundefined" &&
2186                                    arg1 == "showcaptionsetup" && arg2.empty()
2187                                 && arg3 == "%\n \\PassOptionsToPackage{caption=false}{subfig}") {
2188                                 ; // do nothing
2189                         } else if (!in_lyx_preamble) {
2190                                 h_preamble << t.asInput()
2191                                            << '{' << arg1 << '}'
2192                                            << '{' << arg2 << '}'
2193                                            << '{' << arg3 << '}';
2194                         }
2195                         continue;
2196                 }
2197
2198                 if (is_known(t.cs(), known_if_commands)) {
2199                         // must not parse anything in conditional code, since
2200                         // LyX would output the parsed contents unconditionally
2201                         if (!in_lyx_preamble)
2202                                 h_preamble << t.asInput();
2203                         handle_if(p, in_lyx_preamble);
2204                         continue;
2205                 }
2206
2207                 if (!t.cs().empty() && !in_lyx_preamble) {
2208                         h_preamble << '\\' << t.cs();
2209                         continue;
2210                 }
2211         }
2212
2213         // remove the whitespace
2214         p.skip_spaces();
2215
2216         // Force textclass if the user wanted it
2217         if (!forceclass.empty())
2218                 h_textclass = forceclass;
2219         tc.setName(h_textclass);
2220         if (!LayoutFileList::get().haveClass(h_textclass) || !tc.load()) {
2221                 cerr << "Error: Could not read layout file for textclass \"" << h_textclass << "\"." << endl;
2222                 exit(EXIT_FAILURE);
2223         }
2224         if (h_papersides.empty()) {
2225                 ostringstream ss;
2226                 ss << tc.sides();
2227                 h_papersides = ss.str();
2228         }
2229
2230         // If the CJK package is used we cannot set the document language from
2231         // the babel options. Instead, we guess which language is used most
2232         // and set this one.
2233         default_language = h_language;
2234         if (is_full_document &&
2235             (auto_packages.find("CJK") != auto_packages.end() ||
2236              auto_packages.find("CJKutf8") != auto_packages.end())) {
2237                 p.pushPosition();
2238                 h_language = guessLanguage(p, default_language);
2239                 p.popPosition();
2240                 if (explicit_babel && h_language != default_language) {
2241                         // We set the document language to a CJK language,
2242                         // but babel is explicitly called in the user preamble
2243                         // without options. LyX will not add the default
2244                         // language to the document options if it is either
2245                         // english, or no text is set as default language.
2246                         // Therefore we need to add a language option explicitly.
2247                         // FIXME: It would be better to remove all babel calls
2248                         //        from the user preamble, but this is difficult
2249                         //        without re-introducing bug 7861.
2250                         if (h_options.empty())
2251                                 h_options = lyx2babel(default_language);
2252                         else
2253                                 h_options += ',' + lyx2babel(default_language);
2254                 }
2255         }
2256
2257         // Finally, set the quote style.
2258         // LyX knows the following quotes styles:
2259         // british, cjk, cjkangle, danish, english, french, german,
2260         // polish, russian, swedish and swiss
2261         // conversion list taken from
2262         // https://en.wikipedia.org/wiki/Quotation_mark,_non-English_usage
2263         // (quotes for kazakh are unknown)
2264         // british
2265         if (is_known(h_language, known_british_quotes_languages))
2266                 h_quotes_style = "british";
2267         // cjk
2268         else if (is_known(h_language, known_cjk_quotes_languages))
2269                 h_quotes_style = "cjk";
2270         // cjkangle
2271         else if (is_known(h_language, known_cjkangle_quotes_languages))
2272                 h_quotes_style = "cjkangle";
2273         // danish
2274         else if (is_known(h_language, known_danish_quotes_languages))
2275                 h_quotes_style = "danish";
2276         // french
2277         else if (is_known(h_language, known_french_quotes_languages))
2278                 h_quotes_style = "french";
2279         // german
2280         else if (is_known(h_language, known_german_quotes_languages))
2281                 h_quotes_style = "german";
2282         // polish
2283         else if (is_known(h_language, known_polish_quotes_languages))
2284                 h_quotes_style = "polish";
2285         // russian
2286         else if (is_known(h_language, known_russian_quotes_languages))
2287                 h_quotes_style = "russian";
2288         // swedish
2289         else if (is_known(h_language, known_swedish_quotes_languages))
2290                 h_quotes_style = "swedish";
2291         // swiss
2292         else if (is_known(h_language, known_swiss_quotes_languages))
2293                 h_quotes_style = "swiss";
2294         // english
2295         else if (is_known(h_language, known_english_quotes_languages))
2296                 h_quotes_style = "english";
2297 }
2298
2299
2300 string Preamble::parseEncoding(Parser & p, string const & forceclass)
2301 {
2302         TeX2LyXDocClass dummy;
2303         parse(p, forceclass, true, dummy);
2304         if (h_inputencoding != "auto" && h_inputencoding != "default")
2305                 return h_inputencoding;
2306         return "";
2307 }
2308
2309
2310 string babel2lyx(string const & language)
2311 {
2312         char const * const * where = is_known(language, known_languages);
2313         if (where)
2314                 return known_coded_languages[where - known_languages];
2315         return language;
2316 }
2317
2318
2319 string lyx2babel(string const & language)
2320 {
2321         char const * const * where = is_known(language, known_coded_languages);
2322         if (where)
2323                 return known_languages[where - known_coded_languages];
2324         return language;
2325 }
2326
2327
2328 string Preamble::polyglossia2lyx(string const & language)
2329 {
2330         char const * const * where = is_known(language, polyglossia_languages);
2331         if (where)
2332                 return coded_polyglossia_languages[where - polyglossia_languages];
2333         return language;
2334 }
2335
2336
2337 string rgbcolor2code(string const & name)
2338 {
2339         char const * const * where = is_known(name, known_basic_colors);
2340         if (where) {
2341                 // "red", "green" etc
2342                 return known_basic_color_codes[where - known_basic_colors];
2343         }
2344         // "255,0,0", "0,255,0" etc
2345         RGBColor c(RGBColorFromLaTeX(name));
2346         return X11hexname(c);
2347 }
2348
2349 // }])
2350
2351
2352 } // namespace lyx