src/tex2lyx/text.cpp

   1 /**
   2  * \file tex2lyx/text.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Jean-Marc Lasgouttes
   8  * \author Uwe Stöhr
   9  *
  10  * Full author contact details are available in file CREDITS.
  11  */
  12
  13 // {[(
  14
  15 #include <config.h>
  16
  17 #include "tex2lyx.h"
  18
  19 #include "Context.h"
  20 #include "Encoding.h"
  21 #include "FloatList.h"
  22 #include "LaTeXPackages.h"
  23 #include "Layout.h"
  24 #include "Length.h"
  25 #include "Preamble.h"
  26
  27 #include "insets/ExternalTemplate.h"
  28
  29 #include "support/lassert.h"
  30 #include "support/convert.h"
  31 #include "support/FileName.h"
  32 #include "support/filetools.h"
  33 #include "support/lstrings.h"
  34 #include "support/lyxtime.h"
  35
  36 #include <algorithm>
  37 #include <iostream>
  38 #include <map>
  39 #include <sstream>
  40 #include <vector>
  41
  42 using namespace std;
  43 using namespace lyx::support;
  44
  45 namespace lyx {
  46
  47
  48 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  49                 Context const & context, InsetLayout const * layout)
  50 {
  51         bool const forcePlainLayout =
  52                 layout ? layout->forcePlainLayout() : false;
  53         Context newcontext(true, context.textclass);
  54         if (forcePlainLayout)
  55                 newcontext.layout = &context.textclass.plainLayout();
  56         else
  57                 newcontext.font = context.font;
  58         parse_text(p, os, flags, outer, newcontext);
  59         newcontext.check_end_layout(os);
  60 }
  61
  62
  63 namespace {
  64
  65 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  66                 Context const & context, string const & name)
  67 {
  68         InsetLayout const * layout = 0;
  69         DocumentClass::InsetLayouts::const_iterator it =
  70                 context.textclass.insetLayouts().find(from_ascii(name));
  71         if (it != context.textclass.insetLayouts().end())
  72                 layout = &(it->second);
  73         parse_text_in_inset(p, os, flags, outer, context, layout);
  74 }
  75
  76 /// parses a paragraph snippet, useful for example for \\emph{...}
  77 void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
  78                 Context & context)
  79 {
  80         Context newcontext(context);
  81         // Don't inherit the paragraph-level extra stuff
  82         newcontext.par_extra_stuff.clear();
  83         parse_text(p, os, flags, outer, newcontext);
  84         // Make sure that we don't create invalid .lyx files
  85         context.need_layout = newcontext.need_layout;
  86         context.need_end_layout = newcontext.need_end_layout;
  87 }
  88
  89
  90 /*!
  91  * Thin wrapper around parse_text_snippet() using a string.
  92  *
  93  * We completely ignore \c context.need_layout and \c context.need_end_layout,
  94  * because our return value is not used directly (otherwise the stream version
  95  * of parse_text_snippet() could be used). That means that the caller needs
  96  * to do layout management manually.
  97  * This is intended to parse text that does not create any layout changes.
  98  */
  99 string parse_text_snippet(Parser & p, unsigned flags, const bool outer,
 100                   Context & context)
 101 {
 102         Context newcontext(context);
 103         newcontext.need_layout = false;
 104         newcontext.need_end_layout = false;
 105         newcontext.new_layout_allowed = false;
 106         // Avoid warning by Context::~Context()
 107         newcontext.par_extra_stuff.clear();
 108         ostringstream os;
 109         parse_text_snippet(p, os, flags, outer, newcontext);
 110         return os.str();
 111 }
 112
 113
 114 char const * const known_ref_commands[] = { "ref", "pageref", "vref",
 115  "vpageref", "prettyref", "eqref", 0 };
 116
 117 char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref",
 118  "vpageref", "formatted", "eqref", 0 };
 119
 120 /**
 121  * known polyglossia language names (including variants)
 122  */
 123 const char * const polyglossia_languages[] = {
 124 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 125 "nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
 126 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 127 "irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
 128 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 129 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
 130 "brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
 131 "lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 132 "welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
 133 "marathi", "spanish",
 134 "american", "ancient", "australian", "british", "monotonic", "newzealand",
 135 "polytonic", 0};
 136
 137 /**
 138  * the same as polyglossia_languages with .lyx names
 139  * please keep this in sync with polyglossia_languages line by line!
 140  */
 141 const char * const coded_polyglossia_languages[] = {
 142 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 143 "nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
 144 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 145 "irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
 146 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 147 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
 148 "brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
 149 "lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 150 "welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
 151 "marathi", "spanish",
 152 "american", "ancientgreek", "australian", "british", "greek", "newzealand",
 153 "polutonikogreek", 0};
 154
 155 /*!
 156  * natbib commands.
 157  * The starred forms are also known except for "citefullauthor",
 158  * "citeyear" and "citeyearpar".
 159  */
 160 char const * const known_natbib_commands[] = { "cite", "citet", "citep",
 161 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 162 "citefullauthor", "Citet", "Citep", "Citealt", "Citealp", "Citeauthor", 0 };
 163
 164 /*!
 165  * jurabib commands.
 166  * No starred form other than "cite*" known.
 167  */
 168 char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
 169 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 170 // jurabib commands not (yet) supported by LyX:
 171 // "fullcite",
 172 // "footcite", "footcitet", "footcitep", "footcitealt", "footcitealp",
 173 // "footciteauthor", "footciteyear", "footciteyearpar",
 174 "citefield", "citetitle", 0 };
 175
 176 /// LaTeX names for quotes
 177 char const * const known_quotes[] = { "dq", "guillemotleft", "flqq", "og",
 178 "guillemotright", "frqq", "fg", "glq", "glqq", "textquoteleft", "grq", "grqq",
 179 "quotedblbase", "textquotedblleft", "quotesinglbase", "textquoteright", "flq",
 180 "guilsinglleft", "frq", "guilsinglright", 0};
 181
 182 /// the same as known_quotes with .lyx names
 183 char const * const known_coded_quotes[] = { "prd", "ard", "ard", "ard",
 184 "ald", "ald", "ald", "gls", "gld", "els", "els", "grd",
 185 "gld", "grd", "gls", "ers", "fls",
 186 "fls", "frs", "frs", 0};
 187
 188 /// LaTeX names for font sizes
 189 char const * const known_sizes[] = { "tiny", "scriptsize", "footnotesize",
 190 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
 191
 192 /// the same as known_sizes with .lyx names
 193 char const * const known_coded_sizes[] = { "tiny", "scriptsize", "footnotesize",
 194 "small", "normal", "large", "larger", "largest", "huge", "giant", 0};
 195
 196 /// LaTeX 2.09 names for font families
 197 char const * const known_old_font_families[] = { "rm", "sf", "tt", 0};
 198
 199 /// LaTeX names for font families
 200 char const * const known_font_families[] = { "rmfamily", "sffamily",
 201 "ttfamily", 0};
 202
 203 /// LaTeX names for font family changing commands
 204 char const * const known_text_font_families[] = { "textrm", "textsf",
 205 "texttt", 0};
 206
 207 /// The same as known_old_font_families, known_font_families and
 208 /// known_text_font_families with .lyx names
 209 char const * const known_coded_font_families[] = { "roman", "sans",
 210 "typewriter", 0};
 211
 212 /// LaTeX 2.09 names for font series
 213 char const * const known_old_font_series[] = { "bf", 0};
 214
 215 /// LaTeX names for font series
 216 char const * const known_font_series[] = { "bfseries", "mdseries", 0};
 217
 218 /// LaTeX names for font series changing commands
 219 char const * const known_text_font_series[] = { "textbf", "textmd", 0};
 220
 221 /// The same as known_old_font_series, known_font_series and
 222 /// known_text_font_series with .lyx names
 223 char const * const known_coded_font_series[] = { "bold", "medium", 0};
 224
 225 /// LaTeX 2.09 names for font shapes
 226 char const * const known_old_font_shapes[] = { "it", "sl", "sc", 0};
 227
 228 /// LaTeX names for font shapes
 229 char const * const known_font_shapes[] = { "itshape", "slshape", "scshape",
 230 "upshape", 0};
 231
 232 /// LaTeX names for font shape changing commands
 233 char const * const known_text_font_shapes[] = { "textit", "textsl", "textsc",
 234 "textup", 0};
 235
 236 /// The same as known_old_font_shapes, known_font_shapes and
 237 /// known_text_font_shapes with .lyx names
 238 char const * const known_coded_font_shapes[] = { "italic", "slanted",
 239 "smallcaps", "up", 0};
 240
 241 /// Known special characters which need skip_spaces_braces() afterwards
 242 char const * const known_special_chars[] = {"ldots", "lyxarrow",
 243 "textcompwordmark", "slash", 0};
 244
 245 /// the same as known_special_chars with .lyx names
 246 char const * const known_coded_special_chars[] = {"ldots{}", "menuseparator",
 247 "textcompwordmark{}", "slash{}", 0};
 248
 249 /*!
 250  * Graphics file extensions known by the dvips driver of the graphics package.
 251  * These extensions are used to complete the filename of an included
 252  * graphics file if it does not contain an extension.
 253  * The order must be the same that latex uses to find a file, because we
 254  * will use the first extension that matches.
 255  * This is only an approximation for the common cases. If we would want to
 256  * do it right in all cases, we would need to know which graphics driver is
 257  * used and know the extensions of every driver of the graphics package.
 258  */
 259 char const * const known_dvips_graphics_formats[] = {"eps", "ps", "eps.gz",
 260 "ps.gz", "eps.Z", "ps.Z", 0};
 261
 262 /*!
 263  * Graphics file extensions known by the pdftex driver of the graphics package.
 264  * \sa known_dvips_graphics_formats
 265  */
 266 char const * const known_pdftex_graphics_formats[] = {"png", "pdf", "jpg",
 267 "mps", "tif", 0};
 268
 269 /*!
 270  * Known file extensions for TeX files as used by \\include.
 271  */
 272 char const * const known_tex_extensions[] = {"tex", 0};
 273
 274 /// spaces known by InsetSpace
 275 char const * const known_spaces[] = { " ", "space", ",",
 276 "thinspace", "quad", "qquad", "enspace", "enskip",
 277 "negthinspace", "negmedspace", "negthickspace", "textvisiblespace",
 278 "hfill", "dotfill", "hrulefill", "leftarrowfill", "rightarrowfill",
 279 "upbracefill", "downbracefill", 0};
 280
 281 /// the same as known_spaces with .lyx names
 282 char const * const known_coded_spaces[] = { "space{}", "space{}",
 283 "thinspace{}", "thinspace{}", "quad{}", "qquad{}", "enspace{}", "enskip{}",
 284 "negthinspace{}", "negmedspace{}", "negthickspace{}", "textvisiblespace{}",
 285 "hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}", "rightarrowfill{}",
 286 "upbracefill{}", "downbracefill{}", 0};
 287
 288 /// These are translated by LyX to commands like "\\LyX{}", so we have to put
 289 /// them in ERT. "LaTeXe" must come before "LaTeX"!
 290 char const * const known_phrases[] = {"LyX", "TeX", "LaTeXe", "LaTeX", 0};
 291 char const * const known_coded_phrases[] = {"LyX", "TeX", "LaTeX2e", "LaTeX", 0};
 292 int const known_phrase_lengths[] = {3, 5, 7, 0};
 293
 294 // string to store the float type to be able to determine the type of subfloats
 295 string float_type = "";
 296
 297
 298 /// splits "x=z, y=b" into a map and an ordered keyword vector
 299 void split_map(string const & s, map<string, string> & res, vector<string> & keys)
 300 {
 301         vector<string> v;
 302         split(s, v);
 303         res.clear();
 304         keys.resize(v.size());
 305         for (size_t i = 0; i < v.size(); ++i) {
 306                 size_t const pos   = v[i].find('=');
 307                 string const index = trimSpaceAndEol(v[i].substr(0, pos));
 308                 string const value = trimSpaceAndEol(v[i].substr(pos + 1, string::npos));
 309                 res[index] = value;
 310                 keys[i] = index;
 311         }
 312 }
 313
 314
 315 /*!
 316  * Split a LaTeX length into value and unit.
 317  * The latter can be a real unit like "pt", or a latex length variable
 318  * like "\textwidth". The unit may contain additional stuff like glue
 319  * lengths, but we don't care, because such lengths are ERT anyway.
 320  * \returns true if \p value and \p unit are valid.
 321  */
 322 bool splitLatexLength(string const & len, string & value, string & unit)
 323 {
 324         if (len.empty())
 325                 return false;
 326         const string::size_type i = len.find_first_not_of(" -+0123456789.,");
 327         //'4,5' is a valid LaTeX length number. Change it to '4.5'
 328         string const length = subst(len, ',', '.');
 329         if (i == string::npos)
 330                 return false;
 331         if (i == 0) {
 332                 if (len[0] == '\\') {
 333                         // We had something like \textwidth without a factor
 334                         value = "1.0";
 335                 } else {
 336                         return false;
 337                 }
 338         } else {
 339                 value = trimSpaceAndEol(string(length, 0, i));
 340         }
 341         if (value == "-")
 342                 value = "-1.0";
 343         // 'cM' is a valid LaTeX length unit. Change it to 'cm'
 344         if (contains(len, '\\'))
 345                 unit = trimSpaceAndEol(string(len, i));
 346         else
 347                 unit = ascii_lowercase(trimSpaceAndEol(string(len, i)));
 348         return true;
 349 }
 350
 351
 352 /// A simple function to translate a latex length to something LyX can
 353 /// understand. Not perfect, but rather best-effort.
 354 bool translate_len(string const & length, string & valstring, string & unit)
 355 {
 356         if (!splitLatexLength(length, valstring, unit))
 357                 return false;
 358         // LyX uses percent values
 359         double value;
 360         istringstream iss(valstring);
 361         iss >> value;
 362         value *= 100;
 363         ostringstream oss;
 364         oss << value;
 365         string const percentval = oss.str();
 366         // a normal length
 367         if (unit.empty() || unit[0] != '\\')
 368                 return true;
 369         string::size_type const i = unit.find(' ');
 370         string const endlen = (i == string::npos) ? string() : string(unit, i);
 371         if (unit == "\\textwidth") {
 372                 valstring = percentval;
 373                 unit = "text%" + endlen;
 374         } else if (unit == "\\columnwidth") {
 375                 valstring = percentval;
 376                 unit = "col%" + endlen;
 377         } else if (unit == "\\paperwidth") {
 378                 valstring = percentval;
 379                 unit = "page%" + endlen;
 380         } else if (unit == "\\linewidth") {
 381                 valstring = percentval;
 382                 unit = "line%" + endlen;
 383         } else if (unit == "\\paperheight") {
 384                 valstring = percentval;
 385                 unit = "pheight%" + endlen;
 386         } else if (unit == "\\textheight") {
 387                 valstring = percentval;
 388                 unit = "theight%" + endlen;
 389         }
 390         return true;
 391 }
 392
 393 }
 394
 395
 396 string translate_len(string const & length)
 397 {
 398         string unit;
 399         string value;
 400         if (translate_len(length, value, unit))
 401                 return value + unit;
 402         // If the input is invalid, return what we have.
 403         return length;
 404 }
 405
 406
 407 namespace {
 408
 409 /*!
 410  * Translates a LaTeX length into \p value, \p unit and
 411  * \p special parts suitable for a box inset.
 412  * The difference from translate_len() is that a box inset knows about
 413  * some special "units" that are stored in \p special.
 414  */
 415 void translate_box_len(string const & length, string & value, string & unit, string & special)
 416 {
 417         if (translate_len(length, value, unit)) {
 418                 if (unit == "\\height" || unit == "\\depth" ||
 419                     unit == "\\totalheight" || unit == "\\width") {
 420                         special = unit.substr(1);
 421                         // The unit is not used, but LyX requires a dummy setting
 422                         unit = "in";
 423                 } else
 424                         special = "none";
 425         } else {
 426                 value.clear();
 427                 unit = length;
 428                 special = "none";
 429         }
 430 }
 431
 432
 433 /*!
 434  * Find a file with basename \p name in path \p path and an extension
 435  * in \p extensions.
 436  */
 437 string find_file(string const & name, string const & path,
 438                  char const * const * extensions)
 439 {
 440         for (char const * const * what = extensions; *what; ++what) {
 441                 string const trial = addExtension(name, *what);
 442                 if (makeAbsPath(trial, path).exists())
 443                         return trial;
 444         }
 445         return string();
 446 }
 447
 448
 449 void begin_inset(ostream & os, string const & name)
 450 {
 451         os << "\n\\begin_inset " << name;
 452 }
 453
 454
 455 void begin_command_inset(ostream & os, string const & name,
 456                          string const & latexname)
 457 {
 458         begin_inset(os, "CommandInset ");
 459         os << name << "\nLatexCommand " << latexname << '\n';
 460 }
 461
 462
 463 void end_inset(ostream & os)
 464 {
 465         os << "\n\\end_inset\n\n";
 466 }
 467
 468
 469 bool skip_braces(Parser & p)
 470 {
 471         if (p.next_token().cat() != catBegin)
 472                 return false;
 473         p.get_token();
 474         if (p.next_token().cat() == catEnd) {
 475                 p.get_token();
 476                 return true;
 477         }
 478         p.putback();
 479         return false;
 480 }
 481
 482
 483 /// replace LaTeX commands in \p s from the unicodesymbols file with their
 484 /// unicode points
 485 docstring convert_unicodesymbols(docstring s)
 486 {
 487         odocstringstream os;
 488         for (size_t i = 0; i < s.size();) {
 489                 if (s[i] != '\\') {
 490                         os.put(s[i++]);
 491                         continue;
 492                 }
 493                 s = s.substr(i);
 494                 bool termination;
 495                 docstring rem;
 496                 set<string> req;
 497                 docstring parsed = encodings.fromLaTeXCommand(s,
 498                                 Encodings::TEXT_CMD, termination, rem, &req);
 499                 set<string>::const_iterator it = req.begin();
 500                 set<string>::const_iterator en = req.end();
 501                 for (; it != en; ++it)
 502                         preamble.registerAutomaticallyLoadedPackage(*it);
 503                 os << parsed;
 504                 s = rem;
 505                 if (s.empty() || s[0] != '\\')
 506                         i = 0;
 507                 else
 508                         i = 1;
 509         }
 510         return os.str();
 511 }
 512
 513
 514 /// try to convert \p s to a valid InsetCommand argument
 515 string convert_command_inset_arg(string s)
 516 {
 517         if (isAscii(s))
 518                 // since we don't know the input encoding we can't use from_utf8
 519                 s = to_utf8(convert_unicodesymbols(from_ascii(s)));
 520         // LyX cannot handle newlines in a latex command
 521         return subst(s, "\n", " ");
 522 }
 523
 524
 525 void handle_backslash(ostream & os, string const & s)
 526 {
 527         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 528                 if (*it == '\\')
 529                         os << "\n\\backslash\n";
 530                 else
 531                         os << *it;
 532         }
 533 }
 534
 535
 536 void handle_ert(ostream & os, string const & s, Context & context)
 537 {
 538         // We must have a valid layout before outputting the ERT inset.
 539         context.check_layout(os);
 540         Context newcontext(true, context.textclass);
 541         begin_inset(os, "ERT");
 542         os << "\nstatus collapsed\n";
 543         newcontext.check_layout(os);
 544         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 545                 if (*it == '\\')
 546                         os << "\n\\backslash\n";
 547                 else if (*it == '\n') {
 548                         newcontext.new_paragraph(os);
 549                         newcontext.check_layout(os);
 550                 } else
 551                         os << *it;
 552         }
 553         newcontext.check_end_layout(os);
 554         end_inset(os);
 555 }
 556
 557
 558 void handle_comment(ostream & os, string const & s, Context & context)
 559 {
 560         // TODO: Handle this better
 561         Context newcontext(true, context.textclass);
 562         begin_inset(os, "ERT");
 563         os << "\nstatus collapsed\n";
 564         newcontext.check_layout(os);
 565         handle_backslash(os, s);
 566         // make sure that our comment is the last thing on the line
 567         newcontext.new_paragraph(os);
 568         newcontext.check_layout(os);
 569         newcontext.check_end_layout(os);
 570         end_inset(os);
 571 }
 572
 573
 574 Layout const * findLayout(TextClass const & textclass, string const & name, bool command)
 575 {
 576         Layout const * layout = findLayoutWithoutModule(textclass, name, command);
 577         if (layout)
 578                 return layout;
 579         if (checkModule(name, command))
 580                 return findLayoutWithoutModule(textclass, name, command);
 581         return layout;
 582 }
 583
 584
 585 InsetLayout const * findInsetLayout(TextClass const & textclass, string const & name, bool command)
 586 {
 587         InsetLayout const * insetlayout = findInsetLayoutWithoutModule(textclass, name, command);
 588         if (insetlayout)
 589                 return insetlayout;
 590         if (checkModule(name, command))
 591                 return findInsetLayoutWithoutModule(textclass, name, command);
 592         return insetlayout;
 593 }
 594
 595
 596 void eat_whitespace(Parser &, ostream &, Context &, bool);
 597
 598
 599 /*!
 600  * Skips whitespace and braces.
 601  * This should be called after a command has been parsed that is not put into
 602  * ERT, and where LyX adds "{}" if needed.
 603  */
 604 void skip_spaces_braces(Parser & p, bool keepws = false)
 605 {
 606         /* The following four examples produce the same typeset output and
 607            should be handled by this function:
 608            - abc \j{} xyz
 609            - abc \j {} xyz
 610            - abc \j
 611              {} xyz
 612            - abc \j %comment
 613              {} xyz
 614          */
 615         // Unfortunately we need to skip comments, too.
 616         // We can't use eat_whitespace since writing them after the {}
 617         // results in different output in some cases.
 618         bool const skipped_spaces = p.skip_spaces(true);
 619         bool const skipped_braces = skip_braces(p);
 620         if (keepws && skipped_spaces && !skipped_braces)
 621                 // put back the space (it is better handled by check_space)
 622                 p.unskip_spaces(true);
 623 }
 624
 625
 626 void output_command_layout(ostream & os, Parser & p, bool outer,
 627                            Context & parent_context,
 628                            Layout const * newlayout)
 629 {
 630         TeXFont const oldFont = parent_context.font;
 631         // save the current font size
 632         string const size = oldFont.size;
 633         // reset the font size to default, because the font size switches
 634         // don't affect section headings and the like
 635         parent_context.font.size = Context::normalfont.size;
 636         // we only need to write the font change if we have an open layout
 637         if (!parent_context.atParagraphStart())
 638                 output_font_change(os, oldFont, parent_context.font);
 639         parent_context.check_end_layout(os);
 640         Context context(true, parent_context.textclass, newlayout,
 641                         parent_context.layout, parent_context.font);
 642         if (parent_context.deeper_paragraph) {
 643                 // We are beginning a nested environment after a
 644                 // deeper paragraph inside the outer list environment.
 645                 // Therefore we don't need to output a "begin deeper".
 646                 context.need_end_deeper = true;
 647         }
 648         context.check_deeper(os);
 649         context.check_layout(os);
 650         unsigned int optargs = 0;
 651         while (optargs < context.layout->optargs) {
 652                 eat_whitespace(p, os, context, false);
 653                 if (p.next_token().cat() == catEscape ||
 654                     p.next_token().character() != '[')
 655                         break;
 656                 p.get_token(); // eat '['
 657                 begin_inset(os, "Argument\n");
 658                 os << "status collapsed\n\n";
 659                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
 660                 end_inset(os);
 661                 eat_whitespace(p, os, context, false);
 662                 ++optargs;
 663         }
 664         unsigned int reqargs = 0;
 665         while (reqargs < context.layout->reqargs) {
 666                 eat_whitespace(p, os, context, false);
 667                 if (p.next_token().cat() != catBegin)
 668                         break;
 669                 p.get_token(); // eat '{'
 670                 begin_inset(os, "Argument\n");
 671                 os << "status collapsed\n\n";
 672                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
 673                 end_inset(os);
 674                 eat_whitespace(p, os, context, false);
 675                 ++reqargs;
 676         }
 677         parse_text(p, os, FLAG_ITEM, outer, context);
 678         context.check_end_layout(os);
 679         if (parent_context.deeper_paragraph) {
 680                 // We must suppress the "end deeper" because we
 681                 // suppressed the "begin deeper" above.
 682                 context.need_end_deeper = false;
 683         }
 684         context.check_end_deeper(os);
 685         // We don't need really a new paragraph, but
 686         // we must make sure that the next item gets a \begin_layout.
 687         parent_context.new_paragraph(os);
 688         // Set the font size to the original value. No need to output it here
 689         // (Context::begin_layout() will do that if needed)
 690         parent_context.font.size = size;
 691 }
 692
 693
 694 /*!
 695  * Output a space if necessary.
 696  * This function gets called for every whitespace token.
 697  *
 698  * We have three cases here:
 699  * 1. A space must be suppressed. Example: The lyxcode case below
 700  * 2. A space may be suppressed. Example: Spaces before "\par"
 701  * 3. A space must not be suppressed. Example: A space between two words
 702  *
 703  * We currently handle only 1. and 3 and from 2. only the case of
 704  * spaces before newlines as a side effect.
 705  *
 706  * 2. could be used to suppress as many spaces as possible. This has two effects:
 707  * - Reimporting LyX generated LaTeX files changes almost no whitespace
 708  * - Superflous whitespace from non LyX generated LaTeX files is removed.
 709  * The drawback is that the logic inside the function becomes
 710  * complicated, and that is the reason why it is not implemented.
 711  */
 712 void check_space(Parser & p, ostream & os, Context & context)
 713 {
 714         Token const next = p.next_token();
 715         Token const curr = p.curr_token();
 716         // A space before a single newline and vice versa must be ignored
 717         // LyX emits a newline before \end{lyxcode}.
 718         // This newline must be ignored,
 719         // otherwise LyX will add an additional protected space.
 720         if (next.cat() == catSpace ||
 721             next.cat() == catNewline ||
 722             (next.cs() == "end" && context.layout->free_spacing && curr.cat() == catNewline)) {
 723                 return;
 724         }
 725         context.check_layout(os);
 726         os << ' ';
 727 }
 728
 729
 730 /*!
 731  * Parse all arguments of \p command
 732  */
 733 void parse_arguments(string const & command,
 734                      vector<ArgumentType> const & template_arguments,
 735                      Parser & p, ostream & os, bool outer, Context & context)
 736 {
 737         string ert = command;
 738         size_t no_arguments = template_arguments.size();
 739         for (size_t i = 0; i < no_arguments; ++i) {
 740                 switch (template_arguments[i]) {
 741                 case required:
 742                 case req_group:
 743                         // This argument contains regular LaTeX
 744                         handle_ert(os, ert + '{', context);
 745                         eat_whitespace(p, os, context, false);
 746                         if (template_arguments[i] == required)
 747                                 parse_text(p, os, FLAG_ITEM, outer, context);
 748                         else
 749                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 750                         ert = "}";
 751                         break;
 752                 case item:
 753                         // This argument consists only of a single item.
 754                         // The presence of '{' or not must be preserved.
 755                         p.skip_spaces();
 756                         if (p.next_token().cat() == catBegin)
 757                                 ert += '{' + p.verbatim_item() + '}';
 758                         else
 759                                 ert += p.verbatim_item();
 760                         break;
 761                 case displaymath:
 762                 case verbatim:
 763                         // This argument may contain special characters
 764                         ert += '{' + p.verbatim_item() + '}';
 765                         break;
 766                 case optional:
 767                 case opt_group:
 768                         // true because we must not eat whitespace
 769                         // if an optional arg follows we must not strip the
 770                         // brackets from this one
 771                         if (i < no_arguments - 1 &&
 772                             template_arguments[i+1] == optional)
 773                                 ert += p.getFullOpt(true);
 774                         else
 775                                 ert += p.getOpt(true);
 776                         break;
 777                 }
 778         }
 779         handle_ert(os, ert, context);
 780 }
 781
 782
 783 /*!
 784  * Check whether \p command is a known command. If yes,
 785  * handle the command with all arguments.
 786  * \return true if the command was parsed, false otherwise.
 787  */
 788 bool parse_command(string const & command, Parser & p, ostream & os,
 789                    bool outer, Context & context)
 790 {
 791         if (known_commands.find(command) != known_commands.end()) {
 792                 parse_arguments(command, known_commands[command], p, os,
 793                                 outer, context);
 794                 return true;
 795         }
 796         return false;
 797 }
 798
 799
 800 /// Parses a minipage or parbox
 801 void parse_box(Parser & p, ostream & os, unsigned outer_flags,
 802                unsigned inner_flags, bool outer, Context & parent_context,
 803                string const & outer_type, string const & special,
 804                string const & inner_type)
 805 {
 806         string position;
 807         string inner_pos;
 808         string hor_pos = "c";
 809         // We need to set the height to the LaTeX default of 1\\totalheight
 810         // for the case when no height argument is given
 811         string height_value = "1";
 812         string height_unit = "in";
 813         string height_special = "totalheight";
 814         string latex_height;
 815         string width_value;
 816         string width_unit;
 817         string latex_width;
 818         string width_special = "none";
 819         if (!inner_type.empty() && p.hasOpt()) {
 820                 if (inner_type != "makebox")
 821                         position = p.getArg('[', ']');
 822                 else {
 823                         latex_width = p.getArg('[', ']');
 824                         translate_box_len(latex_width, width_value, width_unit, width_special);
 825                         position = "t";
 826                 }
 827                 if (position != "t" && position != "c" && position != "b") {
 828                         cerr << "invalid position " << position << " for "
 829                              << inner_type << endl;
 830                         position = "c";
 831                 }
 832                 if (p.hasOpt()) {
 833                         if (inner_type != "makebox") {
 834                                 latex_height = p.getArg('[', ']');
 835                                 translate_box_len(latex_height, height_value, height_unit, height_special);
 836                         } else
 837                                 hor_pos = p.getArg('[', ']');
 838
 839                         if (p.hasOpt()) {
 840                                 inner_pos = p.getArg('[', ']');
 841                                 if (inner_pos != "c" && inner_pos != "t" &&
 842                                     inner_pos != "b" && inner_pos != "s") {
 843                                         cerr << "invalid inner_pos "
 844                                              << inner_pos << " for "
 845                                              << inner_type << endl;
 846                                         inner_pos = position;
 847                                 }
 848                         }
 849                 }
 850         }
 851         if (inner_type.empty()) {
 852                 if (special.empty() && outer_type != "framebox")
 853                         latex_width = "1\\columnwidth";
 854                 else {
 855                         Parser p2(special);
 856                         latex_width = p2.getArg('[', ']');
 857                         string const opt = p2.getArg('[', ']');
 858                         if (!opt.empty()) {
 859                                 hor_pos = opt;
 860                                 if (hor_pos != "l" && hor_pos != "c" &&
 861                                     hor_pos != "r") {
 862                                         cerr << "invalid hor_pos " << hor_pos
 863                                              << " for " << outer_type << endl;
 864                                         hor_pos = "c";
 865                                 }
 866                         }
 867                 }
 868         } else if (inner_type != "makebox")
 869                 latex_width = p.verbatim_item();
 870         // if e.g. only \ovalbox{content} was used, set the width to 1\columnwidth
 871         // as this is LyX's standard for such cases (except for makebox)
 872         // \framebox is more special and handled below
 873         if (latex_width.empty() && inner_type != "makebox"
 874                 && outer_type != "framebox")
 875                 latex_width = "1\\columnwidth";
 876
 877         translate_len(latex_width, width_value, width_unit);
 878
 879         bool shadedparbox = false;
 880         if (inner_type == "shaded") {
 881                 eat_whitespace(p, os, parent_context, false);
 882                 if (outer_type == "parbox") {
 883                         // Eat '{'
 884                         if (p.next_token().cat() == catBegin)
 885                                 p.get_token();
 886                         eat_whitespace(p, os, parent_context, false);
 887                         shadedparbox = true;
 888                 }
 889                 p.get_token();
 890                 p.getArg('{', '}');
 891         }
 892         // If we already read the inner box we have to push the inner env
 893         if (!outer_type.empty() && !inner_type.empty() &&
 894             (inner_flags & FLAG_END))
 895                 active_environments.push_back(inner_type);
 896         // LyX can't handle length variables
 897         bool use_ert = contains(width_unit, '\\') || contains(height_unit, '\\');
 898         if (!use_ert && !outer_type.empty() && !inner_type.empty()) {
 899                 // Look whether there is some content after the end of the
 900                 // inner box, but before the end of the outer box.
 901                 // If yes, we need to output ERT.
 902                 p.pushPosition();
 903                 if (inner_flags & FLAG_END)
 904                         p.verbatimEnvironment(inner_type);
 905                 else
 906                         p.verbatim_item();
 907                 p.skip_spaces(true);
 908                 bool const outer_env(outer_type == "framed" || outer_type == "minipage");
 909                 if ((outer_env && p.next_token().asInput() != "\\end") ||
 910                     (!outer_env && p.next_token().cat() != catEnd)) {
 911                         // something is between the end of the inner box and
 912                         // the end of the outer box, so we need to use ERT.
 913                         use_ert = true;
 914                 }
 915                 p.popPosition();
 916         }
 917         // if only \makebox{content} was used we can set its width to 1\width
 918         // because this identic and also identic to \mbox
 919         // this doesn't work for \framebox{content}, thus we have to use ERT for this
 920         if (latex_width.empty() && inner_type == "makebox") {
 921                 width_value = "1";
 922                 width_unit = "in";
 923                 width_special = "width";
 924         } else if (latex_width.empty() && outer_type == "framebox") {
 925                 use_ert = true;
 926         }
 927         if (use_ert) {
 928                 ostringstream ss;
 929                 if (!outer_type.empty()) {
 930                         if (outer_flags & FLAG_END)
 931                                 ss << "\\begin{" << outer_type << '}';
 932                         else {
 933                                 ss << '\\' << outer_type << '{';
 934                                 if (!special.empty())
 935                                         ss << special;
 936                         }
 937                 }
 938                 if (!inner_type.empty()) {
 939                         if (inner_type != "shaded") {
 940                                 if (inner_flags & FLAG_END)
 941                                         ss << "\\begin{" << inner_type << '}';
 942                                 else
 943                                         ss << '\\' << inner_type;
 944                         }
 945                         if (!position.empty())
 946                                 ss << '[' << position << ']';
 947                         if (!latex_height.empty())
 948                                 ss << '[' << latex_height << ']';
 949                         if (!inner_pos.empty())
 950                                 ss << '[' << inner_pos << ']';
 951                         ss << '{' << latex_width << '}';
 952                         if (!(inner_flags & FLAG_END))
 953                                 ss << '{';
 954                 }
 955                 if (inner_type == "shaded")
 956                         ss << "\\begin{shaded}";
 957                 handle_ert(os, ss.str(), parent_context);
 958                 if (!inner_type.empty()) {
 959                         parse_text(p, os, inner_flags, outer, parent_context);
 960                         if (inner_flags & FLAG_END)
 961                                 handle_ert(os, "\\end{" + inner_type + '}',
 962                                            parent_context);
 963                         else
 964                                 handle_ert(os, "}", parent_context);
 965                 }
 966                 if (!outer_type.empty()) {
 967                         // If we already read the inner box we have to pop
 968                         // the inner env
 969                         if (!inner_type.empty() && (inner_flags & FLAG_END))
 970                                 active_environments.pop_back();
 971
 972                         // Ensure that the end of the outer box is parsed correctly:
 973                         // The opening brace has been eaten by parse_outer_box()
 974                         if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
 975                                 outer_flags &= ~FLAG_ITEM;
 976                                 outer_flags |= FLAG_BRACE_LAST;
 977                         }
 978                         parse_text(p, os, outer_flags, outer, parent_context);
 979                         if (outer_flags & FLAG_END)
 980                                 handle_ert(os, "\\end{" + outer_type + '}',
 981                                            parent_context);
 982                         else if (inner_type.empty() && outer_type == "framebox")
 983                                 // in this case it is already closed later
 984                                 ;
 985                         else
 986                                 handle_ert(os, "}", parent_context);
 987                 }
 988         } else {
 989                 // LyX does not like empty positions, so we have
 990                 // to set them to the LaTeX default values here.
 991                 if (position.empty())
 992                         position = "c";
 993                 if (inner_pos.empty())
 994                         inner_pos = position;
 995                 parent_context.check_layout(os);
 996                 begin_inset(os, "Box ");
 997                 if (outer_type == "framed")
 998                         os << "Framed\n";
 999                 else if (outer_type == "framebox")
1000                         os << "Boxed\n";
1001                 else if (outer_type == "shadowbox")
1002                         os << "Shadowbox\n";
1003                 else if ((outer_type == "shaded" && inner_type.empty()) ||
1004                              (outer_type == "minipage" && inner_type == "shaded") ||
1005                              (outer_type == "parbox" && inner_type == "shaded")) {
1006                         os << "Shaded\n";
1007                         preamble.registerAutomaticallyLoadedPackage("color");
1008                 } else if (outer_type == "doublebox")
1009                         os << "Doublebox\n";
1010                 else if (outer_type.empty())
1011                         os << "Frameless\n";
1012                 else
1013                         os << outer_type << '\n';
1014                 os << "position \"" << position << "\"\n";
1015                 os << "hor_pos \"" << hor_pos << "\"\n";
1016                 os << "has_inner_box " << !inner_type.empty() << "\n";
1017                 os << "inner_pos \"" << inner_pos << "\"\n";
1018                 os << "use_parbox " << (inner_type == "parbox" || shadedparbox)
1019                    << '\n';
1020                 os << "use_makebox " << (inner_type == "makebox") << '\n';
1021                 os << "width \"" << width_value << width_unit << "\"\n";
1022                 os << "special \"" << width_special << "\"\n";
1023                 os << "height \"" << height_value << height_unit << "\"\n";
1024                 os << "height_special \"" << height_special << "\"\n";
1025                 os << "status open\n\n";
1026
1027                 // Unfortunately we can't use parse_text_in_inset:
1028                 // InsetBox::forcePlainLayout() is hard coded and does not
1029                 // use the inset layout. Apart from that do we call parse_text
1030                 // up to two times, but need only one check_end_layout.
1031                 bool const forcePlainLayout =
1032                         (!inner_type.empty() || inner_type == "makebox") &&
1033                         outer_type != "shaded" && outer_type != "framed";
1034                 Context context(true, parent_context.textclass);
1035                 if (forcePlainLayout)
1036                         context.layout = &context.textclass.plainLayout();
1037                 else
1038                         context.font = parent_context.font;
1039
1040                 // If we have no inner box the contents will be read with the outer box
1041                 if (!inner_type.empty())
1042                         parse_text(p, os, inner_flags, outer, context);
1043
1044                 // Ensure that the end of the outer box is parsed correctly:
1045                 // The opening brace has been eaten by parse_outer_box()
1046                 if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
1047                         outer_flags &= ~FLAG_ITEM;
1048                         outer_flags |= FLAG_BRACE_LAST;
1049                 }
1050
1051                 // Find end of outer box, output contents if inner_type is
1052                 // empty and output possible comments
1053                 if (!outer_type.empty()) {
1054                         // If we already read the inner box we have to pop
1055                         // the inner env
1056                         if (!inner_type.empty() && (inner_flags & FLAG_END))
1057                                 active_environments.pop_back();
1058                         // This does not output anything but comments if
1059                         // inner_type is not empty (see use_ert)
1060                         parse_text(p, os, outer_flags, outer, context);
1061                 }
1062
1063                 context.check_end_layout(os);
1064                 end_inset(os);
1065 #ifdef PRESERVE_LAYOUT
1066                 // LyX puts a % after the end of the minipage
1067                 if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) {
1068                         // new paragraph
1069                         //handle_comment(os, "%dummy", parent_context);
1070                         p.get_token();
1071                         p.skip_spaces();
1072                         parent_context.new_paragraph(os);
1073                 }
1074                 else if (p.next_token().cat() == catSpace || p.next_token().cat() == catNewline) {
1075                         //handle_comment(os, "%dummy", parent_context);
1076                         p.get_token();
1077                         p.skip_spaces();
1078                         // We add a protected space if something real follows
1079                         if (p.good() && p.next_token().cat() != catComment) {
1080                                 begin_inset(os, "space ~\n");
1081                                 end_inset(os);
1082                         }
1083                 }
1084 #endif
1085         }
1086 }
1087
1088
1089 void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer,
1090                      Context & parent_context, string const & outer_type,
1091                      string const & special)
1092 {
1093         eat_whitespace(p, os, parent_context, false);
1094         if (flags & FLAG_ITEM) {
1095                 // Eat '{'
1096                 if (p.next_token().cat() == catBegin)
1097                         p.get_token();
1098                 else
1099                         cerr << "Warning: Ignoring missing '{' after \\"
1100                              << outer_type << '.' << endl;
1101                 eat_whitespace(p, os, parent_context, false);
1102         }
1103         string inner;
1104         unsigned int inner_flags = 0;
1105         p.pushPosition();
1106         if (outer_type == "minipage" || outer_type == "parbox") {
1107                 p.skip_spaces(true);
1108                 while (p.hasOpt()) {
1109                         p.getArg('[', ']');
1110                         p.skip_spaces(true);
1111                 }
1112                 p.getArg('{', '}');
1113                 p.skip_spaces(true);
1114                 if (outer_type == "parbox") {
1115                         // Eat '{'
1116                         if (p.next_token().cat() == catBegin)
1117                                 p.get_token();
1118                         p.skip_spaces(true);
1119                 }
1120         }
1121         if (outer_type == "shaded") {
1122                 // These boxes never have an inner box
1123                 ;
1124         } else if (p.next_token().asInput() == "\\parbox") {
1125                 inner = p.get_token().cs();
1126                 inner_flags = FLAG_ITEM;
1127         } else if (p.next_token().asInput() == "\\begin") {
1128                 // Is this a minipage or shaded box?
1129                 p.pushPosition();
1130                 p.get_token();
1131                 inner = p.getArg('{', '}');
1132                 p.popPosition();
1133                 if (inner == "minipage" || inner == "shaded")
1134                         inner_flags = FLAG_END;
1135                 else
1136                         inner = "";
1137         }
1138         p.popPosition();
1139         if (inner_flags == FLAG_END) {
1140                 if (inner != "shaded")
1141                 {
1142                         p.get_token();
1143                         p.getArg('{', '}');
1144                         eat_whitespace(p, os, parent_context, false);
1145                 }
1146                 parse_box(p, os, flags, FLAG_END, outer, parent_context,
1147                           outer_type, special, inner);
1148         } else {
1149                 if (inner_flags == FLAG_ITEM) {
1150                         p.get_token();
1151                         eat_whitespace(p, os, parent_context, false);
1152                 }
1153                 parse_box(p, os, flags, inner_flags, outer, parent_context,
1154                           outer_type, special, inner);
1155         }
1156 }
1157
1158
1159 void parse_listings(Parser & p, ostream & os, Context & parent_context, bool in_line)
1160 {
1161         parent_context.check_layout(os);
1162         begin_inset(os, "listings\n");
1163         if (p.hasOpt()) {
1164                 string arg = p.verbatimOption();
1165                 os << "lstparams " << '"' << arg << '"' << '\n';
1166         }
1167         if (in_line)
1168                 os << "inline true\n";
1169         else
1170                 os << "inline false\n";
1171         os << "status collapsed\n";
1172         Context context(true, parent_context.textclass);
1173         context.layout = &parent_context.textclass.plainLayout();
1174         string s;
1175         if (in_line) {
1176                 s = p.plainCommand('!', '!', "lstinline");
1177                 context.new_paragraph(os);
1178                 context.check_layout(os);
1179         } else
1180                 s = p.plainEnvironment("lstlisting");
1181         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1182                 if (*it == '\\')
1183                         os << "\n\\backslash\n";
1184                 else if (*it == '\n') {
1185                         // avoid adding an empty paragraph at the end
1186                         if (it + 1 != et) {
1187                                 context.new_paragraph(os);
1188                                 context.check_layout(os);
1189                         }
1190                 } else
1191                         os << *it;
1192         }
1193         context.check_end_layout(os);
1194         end_inset(os);
1195 }
1196
1197
1198 /// parse an unknown environment
1199 void parse_unknown_environment(Parser & p, string const & name, ostream & os,
1200                                unsigned flags, bool outer,
1201                                Context & parent_context)
1202 {
1203         if (name == "tabbing")
1204                 // We need to remember that we have to handle '\=' specially
1205                 flags |= FLAG_TABBING;
1206
1207         // We need to translate font changes and paragraphs inside the
1208         // environment to ERT if we have a non standard font.
1209         // Otherwise things like
1210         // \large\begin{foo}\huge bar\end{foo}
1211         // will not work.
1212         bool const specialfont =
1213                 (parent_context.font != parent_context.normalfont);
1214         bool const new_layout_allowed = parent_context.new_layout_allowed;
1215         if (specialfont)
1216                 parent_context.new_layout_allowed = false;
1217         handle_ert(os, "\\begin{" + name + "}", parent_context);
1218         parse_text_snippet(p, os, flags, outer, parent_context);
1219         handle_ert(os, "\\end{" + name + "}", parent_context);
1220         if (specialfont)
1221                 parent_context.new_layout_allowed = new_layout_allowed;
1222 }
1223
1224
1225 void parse_environment(Parser & p, ostream & os, bool outer,
1226                        string & last_env, Context & parent_context)
1227 {
1228         Layout const * newlayout;
1229         InsetLayout const * newinsetlayout = 0;
1230         string const name = p.getArg('{', '}');
1231         const bool is_starred = suffixIs(name, '*');
1232         string const unstarred_name = rtrim(name, "*");
1233         active_environments.push_back(name);
1234
1235         if (is_math_env(name)) {
1236                 parent_context.check_layout(os);
1237                 begin_inset(os, "Formula ");
1238                 os << "\\begin{" << name << "}";
1239                 parse_math(p, os, FLAG_END, MATH_MODE);
1240                 os << "\\end{" << name << "}";
1241                 end_inset(os);
1242                 if (is_display_math_env(name)) {
1243                         // Prevent the conversion of a line break to a space
1244                         // (bug 7668). This does not change the output, but
1245                         // looks ugly in LyX.
1246                         eat_whitespace(p, os, parent_context, false);
1247                 }
1248         }
1249
1250         else if (is_known(name, polyglossia_languages)) {
1251                 // We must begin a new paragraph if not already done
1252                 if (! parent_context.atParagraphStart()) {
1253                         parent_context.check_end_layout(os);
1254                         parent_context.new_paragraph(os);
1255                 }
1256                 // save the language in the context so that it is
1257                 // handled by parse_text
1258                 parent_context.font.language = polyglossia2lyx(name);
1259                 parse_text(p, os, FLAG_END, outer, parent_context);
1260                 // Just in case the environment is empty
1261                 parent_context.extra_stuff.erase();
1262                 // We must begin a new paragraph to reset the language
1263                 parent_context.new_paragraph(os);
1264                 p.skip_spaces();
1265         }
1266
1267         else if (unstarred_name == "tabular" || name == "longtable") {
1268                 eat_whitespace(p, os, parent_context, false);
1269                 string width = "0pt";
1270                 if (name == "tabular*") {
1271                         width = lyx::translate_len(p.getArg('{', '}'));
1272                         eat_whitespace(p, os, parent_context, false);
1273                 }
1274                 parent_context.check_layout(os);
1275                 begin_inset(os, "Tabular ");
1276                 handle_tabular(p, os, name, width, parent_context);
1277                 end_inset(os);
1278                 p.skip_spaces();
1279         }
1280
1281         else if (parent_context.textclass.floats().typeExist(unstarred_name)) {
1282                 eat_whitespace(p, os, parent_context, false);
1283                 string const opt = p.hasOpt() ? p.getArg('[', ']') : string();
1284                 eat_whitespace(p, os, parent_context, false);
1285                 parent_context.check_layout(os);
1286                 begin_inset(os, "Float " + unstarred_name + "\n");
1287                 // store the float type for subfloats
1288                 // subfloats only work with figures and tables
1289                 if (unstarred_name == "figure")
1290                         float_type = unstarred_name;
1291                 else if (unstarred_name == "table")
1292                         float_type = unstarred_name;
1293                 else
1294                         float_type = "";
1295                 if (!opt.empty())
1296                         os << "placement " << opt << '\n';
1297                 if (contains(opt, "H"))
1298                         preamble.registerAutomaticallyLoadedPackage("float");
1299                 else {
1300                         Floating const & fl = parent_context.textclass.floats()
1301                                 .getType(unstarred_name);
1302                         if (!fl.floattype().empty() && fl.usesFloatPkg())
1303                                 preamble.registerAutomaticallyLoadedPackage("float");
1304                 }
1305
1306                 os << "wide " << convert<string>(is_starred)
1307                    << "\nsideways false"
1308                    << "\nstatus open\n\n";
1309                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1310                 end_inset(os);
1311                 // We don't need really a new paragraph, but
1312                 // we must make sure that the next item gets a \begin_layout.
1313                 parent_context.new_paragraph(os);
1314                 p.skip_spaces();
1315                 // the float is parsed thus delete the type
1316                 float_type = "";
1317         }
1318
1319         else if (unstarred_name == "sidewaysfigure"
1320                 || unstarred_name == "sidewaystable") {
1321                 eat_whitespace(p, os, parent_context, false);
1322                 parent_context.check_layout(os);
1323                 if (unstarred_name == "sidewaysfigure")
1324                         begin_inset(os, "Float figure\n");
1325                 else
1326                         begin_inset(os, "Float table\n");
1327                 os << "wide " << convert<string>(is_starred)
1328                    << "\nsideways true"
1329                    << "\nstatus open\n\n";
1330                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1331                 end_inset(os);
1332                 // We don't need really a new paragraph, but
1333                 // we must make sure that the next item gets a \begin_layout.
1334                 parent_context.new_paragraph(os);
1335                 p.skip_spaces();
1336                 preamble.registerAutomaticallyLoadedPackage("rotfloat");
1337         }
1338
1339         else if (name == "wrapfigure" || name == "wraptable") {
1340                 // syntax is \begin{wrapfigure}[lines]{placement}[overhang]{width}
1341                 eat_whitespace(p, os, parent_context, false);
1342                 parent_context.check_layout(os);
1343                 // default values
1344                 string lines = "0";
1345                 string overhang = "0col%";
1346                 // parse
1347                 if (p.hasOpt())
1348                         lines = p.getArg('[', ']');
1349                 string const placement = p.getArg('{', '}');
1350                 if (p.hasOpt())
1351                         overhang = p.getArg('[', ']');
1352                 string const width = p.getArg('{', '}');
1353                 // write
1354                 if (name == "wrapfigure")
1355                         begin_inset(os, "Wrap figure\n");
1356                 else
1357                         begin_inset(os, "Wrap table\n");
1358                 os << "lines " << lines
1359                    << "\nplacement " << placement
1360                    << "\noverhang " << lyx::translate_len(overhang)
1361                    << "\nwidth " << lyx::translate_len(width)
1362                    << "\nstatus open\n\n";
1363                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1364                 end_inset(os);
1365                 // We don't need really a new paragraph, but
1366                 // we must make sure that the next item gets a \begin_layout.
1367                 parent_context.new_paragraph(os);
1368                 p.skip_spaces();
1369                 preamble.registerAutomaticallyLoadedPackage("wrapfig");
1370         }
1371
1372         else if (name == "minipage") {
1373                 eat_whitespace(p, os, parent_context, false);
1374                 // Test whether this is an outer box of a shaded box
1375                 p.pushPosition();
1376                 // swallow arguments
1377                 while (p.hasOpt()) {
1378                         p.getArg('[', ']');
1379                         p.skip_spaces(true);
1380                 }
1381                 p.getArg('{', '}');
1382                 p.skip_spaces(true);
1383                 Token t = p.get_token();
1384                 bool shaded = false;
1385                 if (t.asInput() == "\\begin") {
1386                         p.skip_spaces(true);
1387                         if (p.getArg('{', '}') == "shaded")
1388                                 shaded = true;
1389                 }
1390                 p.popPosition();
1391                 if (shaded)
1392                         parse_outer_box(p, os, FLAG_END, outer,
1393                                         parent_context, name, "shaded");
1394                 else
1395                         parse_box(p, os, 0, FLAG_END, outer, parent_context,
1396                                   "", "", name);
1397                 p.skip_spaces();
1398         }
1399
1400         else if (name == "comment") {
1401                 eat_whitespace(p, os, parent_context, false);
1402                 parent_context.check_layout(os);
1403                 begin_inset(os, "Note Comment\n");
1404                 os << "status open\n";
1405                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1406                 end_inset(os);
1407                 p.skip_spaces();
1408                 skip_braces(p); // eat {} that might by set by LyX behind comments
1409                 preamble.registerAutomaticallyLoadedPackage("verbatim");
1410         }
1411
1412         else if (name == "verbatim") {
1413                 os << "\n\\end_layout\n\n\\begin_layout Verbatim\n";
1414                 string const s = p.plainEnvironment("verbatim");
1415                 string::const_iterator it2 = s.begin();
1416                 for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1417                         if (*it == '\\')
1418                                 os << "\\backslash ";
1419                         else if (*it == '\n') {
1420                                 it2 = it + 1;
1421                                 // avoid adding an empty paragraph at the end
1422                                 // FIXME: if there are 2 consecutive spaces at the end ignore it
1423                                 // because LyX will re-add a \n
1424                                 // This hack must be removed once bug 8049 is fixed!
1425                                 if ((it + 1 != et) && (it + 2 != et || *it2 != '\n'))
1426                                         os << "\n\\end_layout\n\\begin_layout Verbatim\n";
1427                         } else
1428                                 os << *it;
1429                 }
1430                 os << "\n\\end_layout\n\n";
1431                 p.skip_spaces();
1432                 // reset to Standard layout
1433                 os << "\n\\begin_layout Standard\n";
1434         }
1435
1436         else if (name == "lyxgreyedout") {
1437                 eat_whitespace(p, os, parent_context, false);
1438                 parent_context.check_layout(os);
1439                 begin_inset(os, "Note Greyedout\n");
1440                 os << "status open\n";
1441                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1442                 end_inset(os);
1443                 p.skip_spaces();
1444                 if (!preamble.notefontcolor().empty())
1445                         preamble.registerAutomaticallyLoadedPackage("color");
1446         }
1447
1448         else if (name == "framed" || name == "shaded") {
1449                 eat_whitespace(p, os, parent_context, false);
1450                 parse_outer_box(p, os, FLAG_END, outer, parent_context, name, "");
1451                 p.skip_spaces();
1452         }
1453
1454         else if (name == "lstlisting") {
1455                 eat_whitespace(p, os, parent_context, false);
1456                 // FIXME handle the automatic color package loading
1457                 // uwestoehr asks: In what case color is loaded?
1458                 parse_listings(p, os, parent_context, false);
1459                 p.skip_spaces();
1460         }
1461
1462         else if (!parent_context.new_layout_allowed)
1463                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1464                                           parent_context);
1465
1466         // Alignment and spacing settings
1467         // FIXME (bug xxxx): These settings can span multiple paragraphs and
1468         //                                       therefore are totally broken!
1469         // Note that \centering, raggedright, and raggedleft cannot be handled, as
1470         // they are commands not environments. They are furthermore switches that
1471         // can be ended by another switches, but also by commands like \footnote or
1472         // \parbox. So the only safe way is to leave them untouched.
1473         else if (name == "center" || name == "centering" ||
1474                  name == "flushleft" || name == "flushright" ||
1475                  name == "singlespace" || name == "onehalfspace" ||
1476                  name == "doublespace" || name == "spacing") {
1477                 eat_whitespace(p, os, parent_context, false);
1478                 // We must begin a new paragraph if not already done
1479                 if (! parent_context.atParagraphStart()) {
1480                         parent_context.check_end_layout(os);
1481                         parent_context.new_paragraph(os);
1482                 }
1483                 if (name == "flushleft")
1484                         parent_context.add_extra_stuff("\\align left\n");
1485                 else if (name == "flushright")
1486                         parent_context.add_extra_stuff("\\align right\n");
1487                 else if (name == "center" || name == "centering")
1488                         parent_context.add_extra_stuff("\\align center\n");
1489                 else if (name == "singlespace")
1490                         parent_context.add_extra_stuff("\\paragraph_spacing single\n");
1491                 else if (name == "onehalfspace") {
1492                         parent_context.add_extra_stuff("\\paragraph_spacing onehalf\n");
1493                         preamble.registerAutomaticallyLoadedPackage("setspace");
1494                 } else if (name == "doublespace") {
1495                         parent_context.add_extra_stuff("\\paragraph_spacing double\n");
1496                         preamble.registerAutomaticallyLoadedPackage("setspace");
1497                 } else if (name == "spacing") {
1498                         parent_context.add_extra_stuff("\\paragraph_spacing other " + p.verbatim_item() + "\n");
1499                         preamble.registerAutomaticallyLoadedPackage("setspace");
1500                 }
1501                 parse_text(p, os, FLAG_END, outer, parent_context);
1502                 // Just in case the environment is empty
1503                 parent_context.extra_stuff.erase();
1504                 // We must begin a new paragraph to reset the alignment
1505                 parent_context.new_paragraph(os);
1506                 p.skip_spaces();
1507         }
1508
1509         // The single '=' is meant here.
1510         else if ((newlayout = findLayout(parent_context.textclass, name, false))) {
1511                 eat_whitespace(p, os, parent_context, false);
1512                 Context context(true, parent_context.textclass, newlayout,
1513                                 parent_context.layout, parent_context.font);
1514                 if (parent_context.deeper_paragraph) {
1515                         // We are beginning a nested environment after a
1516                         // deeper paragraph inside the outer list environment.
1517                         // Therefore we don't need to output a "begin deeper".
1518                         context.need_end_deeper = true;
1519                 }
1520                 parent_context.check_end_layout(os);
1521                 if (last_env == name) {
1522                         // we need to output a separator since LyX would export
1523                         // the two environments as one otherwise (bug 5716)
1524                         docstring const sep = from_ascii("--Separator--");
1525                         TeX2LyXDocClass const & textclass(parent_context.textclass);
1526                         if (textclass.hasLayout(sep)) {
1527                                 Context newcontext(parent_context);
1528                                 newcontext.layout = &(textclass[sep]);
1529                                 newcontext.check_layout(os);
1530                                 newcontext.check_end_layout(os);
1531                         } else {
1532                                 parent_context.check_layout(os);
1533                                 begin_inset(os, "Note Note\n");
1534                                 os << "status closed\n";
1535                                 Context newcontext(true, textclass,
1536                                                 &(textclass.defaultLayout()));
1537                                 newcontext.check_layout(os);
1538                                 newcontext.check_end_layout(os);
1539                                 end_inset(os);
1540                                 parent_context.check_end_layout(os);
1541                         }
1542                 }
1543                 switch (context.layout->latextype) {
1544                 case  LATEX_LIST_ENVIRONMENT:
1545                         context.add_par_extra_stuff("\\labelwidthstring "
1546                                                     + p.verbatim_item() + '\n');
1547                         p.skip_spaces();
1548                         break;
1549                 case  LATEX_BIB_ENVIRONMENT:
1550                         p.verbatim_item(); // swallow next arg
1551                         p.skip_spaces();
1552                         break;
1553                 default:
1554                         break;
1555                 }
1556                 context.check_deeper(os);
1557                 // handle known optional and required arguments
1558                 // layouts require all optional arguments before the required ones
1559                 // Unfortunately LyX can't handle arguments of list arguments (bug 7468):
1560                 // It is impossible to place anything after the environment name,
1561                 // but before the first \\item.
1562                 if (context.layout->latextype == LATEX_ENVIRONMENT) {
1563                         bool need_layout = true;
1564                         unsigned int optargs = 0;
1565                         while (optargs < context.layout->optargs) {
1566                                 eat_whitespace(p, os, context, false);
1567                                 if (p.next_token().cat() == catEscape ||
1568                                     p.next_token().character() != '[')
1569                                         break;
1570                                 p.get_token(); // eat '['
1571                                 if (need_layout) {
1572                                         context.check_layout(os);
1573                                         need_layout = false;
1574                                 }
1575                                 begin_inset(os, "Argument\n");
1576                                 os << "status collapsed\n\n";
1577                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
1578                                 end_inset(os);
1579                                 eat_whitespace(p, os, context, false);
1580                                 ++optargs;
1581                         }
1582                         unsigned int reqargs = 0;
1583                         while (reqargs < context.layout->reqargs) {
1584                                 eat_whitespace(p, os, context, false);
1585                                 if (p.next_token().cat() != catBegin)
1586                                         break;
1587                                 p.get_token(); // eat '{'
1588                                 if (need_layout) {
1589                                         context.check_layout(os);
1590                                         need_layout = false;
1591                                 }
1592                                 begin_inset(os, "Argument\n");
1593                                 os << "status collapsed\n\n";
1594                                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
1595                                 end_inset(os);
1596                                 eat_whitespace(p, os, context, false);
1597                                 ++reqargs;
1598                         }
1599                 }
1600                 parse_text(p, os, FLAG_END, outer, context);
1601                 context.check_end_layout(os);
1602                 if (parent_context.deeper_paragraph) {
1603                         // We must suppress the "end deeper" because we
1604                         // suppressed the "begin deeper" above.
1605                         context.need_end_deeper = false;
1606                 }
1607                 context.check_end_deeper(os);
1608                 parent_context.new_paragraph(os);
1609                 p.skip_spaces();
1610                 if (!preamble.titleLayoutFound())
1611                         preamble.titleLayoutFound(newlayout->intitle);
1612                 set<string> const & req = newlayout->requires();
1613                 set<string>::const_iterator it = req.begin();
1614                 set<string>::const_iterator en = req.end();
1615                 for (; it != en; ++it)
1616                         preamble.registerAutomaticallyLoadedPackage(*it);
1617         }
1618
1619         // The single '=' is meant here.
1620         else if ((newinsetlayout = findInsetLayout(parent_context.textclass, name, false))) {
1621                 eat_whitespace(p, os, parent_context, false);
1622                 parent_context.check_layout(os);
1623                 begin_inset(os, "Flex ");
1624                 os << to_utf8(newinsetlayout->name()) << '\n'
1625                    << "status collapsed\n";
1626                 parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
1627                 end_inset(os);
1628         }
1629
1630         else if (name == "appendix") {
1631                 // This is no good latex style, but it works and is used in some documents...
1632                 eat_whitespace(p, os, parent_context, false);
1633                 parent_context.check_end_layout(os);
1634                 Context context(true, parent_context.textclass, parent_context.layout,
1635                                 parent_context.layout, parent_context.font);
1636                 context.check_layout(os);
1637                 os << "\\start_of_appendix\n";
1638                 parse_text(p, os, FLAG_END, outer, context);
1639                 context.check_end_layout(os);
1640                 p.skip_spaces();
1641         }
1642
1643         else if (known_environments.find(name) != known_environments.end()) {
1644                 vector<ArgumentType> arguments = known_environments[name];
1645                 // The last "argument" denotes wether we may translate the
1646                 // environment contents to LyX
1647                 // The default required if no argument is given makes us
1648                 // compatible with the reLyXre environment.
1649                 ArgumentType contents = arguments.empty() ?
1650                         required :
1651                         arguments.back();
1652                 if (!arguments.empty())
1653                         arguments.pop_back();
1654                 // See comment in parse_unknown_environment()
1655                 bool const specialfont =
1656                         (parent_context.font != parent_context.normalfont);
1657                 bool const new_layout_allowed =
1658                         parent_context.new_layout_allowed;
1659                 if (specialfont)
1660                         parent_context.new_layout_allowed = false;
1661                 parse_arguments("\\begin{" + name + "}", arguments, p, os,
1662                                 outer, parent_context);
1663                 if (contents == verbatim)
1664                         handle_ert(os, p.verbatimEnvironment(name),
1665                                    parent_context);
1666                 else
1667                         parse_text_snippet(p, os, FLAG_END, outer,
1668                                            parent_context);
1669                 handle_ert(os, "\\end{" + name + "}", parent_context);
1670                 if (specialfont)
1671                         parent_context.new_layout_allowed = new_layout_allowed;
1672         }
1673
1674         else
1675                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1676                                           parent_context);
1677
1678         last_env = name;
1679         active_environments.pop_back();
1680 }
1681
1682
1683 /// parses a comment and outputs it to \p os.
1684 void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
1685 {
1686         LASSERT(t.cat() == catComment, return);
1687         if (!t.cs().empty()) {
1688                 context.check_layout(os);
1689                 handle_comment(os, '%' + t.cs(), context);
1690                 if (p.next_token().cat() == catNewline) {
1691                         // A newline after a comment line starts a new
1692                         // paragraph
1693                         if (context.new_layout_allowed) {
1694                                 if(!context.atParagraphStart())
1695                                         // Only start a new paragraph if not already
1696                                         // done (we might get called recursively)
1697                                         context.new_paragraph(os);
1698                         } else
1699                                 handle_ert(os, "\n", context);
1700                         eat_whitespace(p, os, context, true);
1701                 }
1702         } else {
1703                 // "%\n" combination
1704                 p.skip_spaces();
1705         }
1706 }
1707
1708
1709 /*!
1710  * Reads spaces and comments until the first non-space, non-comment token.
1711  * New paragraphs (double newlines or \\par) are handled like simple spaces
1712  * if \p eatParagraph is true.
1713  * Spaces are skipped, but comments are written to \p os.
1714  */
1715 void eat_whitespace(Parser & p, ostream & os, Context & context,
1716                     bool eatParagraph)
1717 {
1718         while (p.good()) {
1719                 Token const & t = p.get_token();
1720                 if (t.cat() == catComment)
1721                         parse_comment(p, os, t, context);
1722                 else if ((! eatParagraph && p.isParagraph()) ||
1723                          (t.cat() != catSpace && t.cat() != catNewline)) {
1724                         p.putback();
1725                         return;
1726                 }
1727         }
1728 }
1729
1730
1731 /*!
1732  * Set a font attribute, parse text and reset the font attribute.
1733  * \param attribute Attribute name (e.g. \\family, \\shape etc.)
1734  * \param currentvalue Current value of the attribute. Is set to the new
1735  * value during parsing.
1736  * \param newvalue New value of the attribute
1737  */
1738 void parse_text_attributes(Parser & p, ostream & os, unsigned flags, bool outer,
1739                            Context & context, string const & attribute,
1740                            string & currentvalue, string const & newvalue)
1741 {
1742         context.check_layout(os);
1743         string const oldvalue = currentvalue;
1744         currentvalue = newvalue;
1745         os << '\n' << attribute << ' ' << newvalue << "\n";
1746         parse_text_snippet(p, os, flags, outer, context);
1747         context.check_layout(os);
1748         os << '\n' << attribute << ' ' << oldvalue << "\n";
1749         currentvalue = oldvalue;
1750 }
1751
1752
1753 /// get the arguments of a natbib or jurabib citation command
1754 void get_cite_arguments(Parser & p, bool natbibOrder,
1755         string & before, string & after)
1756 {
1757         // We need to distinguish "" and "[]", so we can't use p.getOpt().
1758
1759         // text before the citation
1760         before.clear();
1761         // text after the citation
1762         after = p.getFullOpt();
1763
1764         if (!after.empty()) {
1765                 before = p.getFullOpt();
1766                 if (natbibOrder && !before.empty())
1767                         swap(before, after);
1768         }
1769 }
1770
1771
1772 /// Convert filenames with TeX macros and/or quotes to something LyX
1773 /// can understand
1774 string const normalize_filename(string const & name)
1775 {
1776         Parser p(trim(name, "\""));
1777         ostringstream os;
1778         while (p.good()) {
1779                 Token const & t = p.get_token();
1780                 if (t.cat() != catEscape)
1781                         os << t.asInput();
1782                 else if (t.cs() == "lyxdot") {
1783                         // This is used by LyX for simple dots in relative
1784                         // names
1785                         os << '.';
1786                         p.skip_spaces();
1787                 } else if (t.cs() == "space") {
1788                         os << ' ';
1789                         p.skip_spaces();
1790                 } else
1791                         os << t.asInput();
1792         }
1793         return os.str();
1794 }
1795
1796
1797 /// Convert \p name from TeX convention (relative to master file) to LyX
1798 /// convention (relative to .lyx file) if it is relative
1799 void fix_relative_filename(string & name)
1800 {
1801         if (FileName::isAbsolute(name))
1802                 return;
1803
1804         name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFileName()),
1805                                    from_utf8(getParentFilePath())));
1806 }
1807
1808
1809 /// Parse a NoWeb Scrap section. The initial "<<" is already parsed.
1810 void parse_noweb(Parser & p, ostream & os, Context & context)
1811 {
1812         // assemble the rest of the keyword
1813         string name("<<");
1814         bool scrap = false;
1815         while (p.good()) {
1816                 Token const & t = p.get_token();
1817                 if (t.asInput() == ">" && p.next_token().asInput() == ">") {
1818                         name += ">>";
1819                         p.get_token();
1820                         scrap = (p.good() && p.next_token().asInput() == "=");
1821                         if (scrap)
1822                                 name += p.get_token().asInput();
1823                         break;
1824                 }
1825                 name += t.asInput();
1826         }
1827
1828         if (!scrap || !context.new_layout_allowed ||
1829             !context.textclass.hasLayout(from_ascii("Scrap"))) {
1830                 cerr << "Warning: Could not interpret '" << name
1831                      << "'. Ignoring it." << endl;
1832                 return;
1833         }
1834
1835         // We use new_paragraph instead of check_end_layout because the stuff
1836         // following the noweb chunk needs to start with a \begin_layout.
1837         // This may create a new paragraph even if there was none in the
1838         // noweb file, but the alternative is an invalid LyX file. Since
1839         // noweb code chunks are implemented with a layout style in LyX they
1840         // always must be in an own paragraph.
1841         context.new_paragraph(os);
1842         Context newcontext(true, context.textclass,
1843                 &context.textclass[from_ascii("Scrap")]);
1844         newcontext.check_layout(os);
1845         os << name;
1846         while (p.good()) {
1847                 Token const & t = p.get_token();
1848                 // We abuse the parser a bit, because this is no TeX syntax
1849                 // at all.
1850                 if (t.cat() == catEscape)
1851                         os << subst(t.asInput(), "\\", "\n\\backslash\n");
1852                 else {
1853                         ostringstream oss;
1854                         Context tmp(false, context.textclass,
1855                                     &context.textclass[from_ascii("Scrap")]);
1856                         tmp.need_end_layout = true;
1857                         tmp.check_layout(oss);
1858                         os << subst(t.asInput(), "\n", oss.str());
1859                 }
1860                 // The scrap chunk is ended by an @ at the beginning of a line.
1861                 // After the @ the line may contain a comment and/or
1862                 // whitespace, but nothing else.
1863                 if (t.asInput() == "@" && p.prev_token().cat() == catNewline &&
1864                     (p.next_token().cat() == catSpace ||
1865                      p.next_token().cat() == catNewline ||
1866                      p.next_token().cat() == catComment)) {
1867                         while (p.good() && p.next_token().cat() == catSpace)
1868                                 os << p.get_token().asInput();
1869                         if (p.next_token().cat() == catComment)
1870                                 // The comment includes a final '\n'
1871                                 os << p.get_token().asInput();
1872                         else {
1873                                 if (p.next_token().cat() == catNewline)
1874                                         p.get_token();
1875                                 os << '\n';
1876                         }
1877                         break;
1878                 }
1879         }
1880         newcontext.check_end_layout(os);
1881 }
1882
1883
1884 /// detects \\def, \\long\\def and \\global\\long\\def with ws and comments
1885 bool is_macro(Parser & p)
1886 {
1887         Token first = p.curr_token();
1888         if (first.cat() != catEscape || !p.good())
1889                 return false;
1890         if (first.cs() == "def")
1891                 return true;
1892         if (first.cs() != "global" && first.cs() != "long")
1893                 return false;
1894         Token second = p.get_token();
1895         int pos = 1;
1896         while (p.good() && !p.isParagraph() && (second.cat() == catSpace ||
1897                second.cat() == catNewline || second.cat() == catComment)) {
1898                 second = p.get_token();
1899                 pos++;
1900         }
1901         bool secondvalid = second.cat() == catEscape;
1902         Token third;
1903         bool thirdvalid = false;
1904         if (p.good() && first.cs() == "global" && secondvalid &&
1905             second.cs() == "long") {
1906                 third = p.get_token();
1907                 pos++;
1908                 while (p.good() && !p.isParagraph() &&
1909                        (third.cat() == catSpace ||
1910                         third.cat() == catNewline ||
1911                         third.cat() == catComment)) {
1912                         third = p.get_token();
1913                         pos++;
1914                 }
1915                 thirdvalid = third.cat() == catEscape;
1916         }
1917         for (int i = 0; i < pos; ++i)
1918                 p.putback();
1919         if (!secondvalid)
1920                 return false;
1921         if (!thirdvalid)
1922                 return (first.cs() == "global" || first.cs() == "long") &&
1923                        second.cs() == "def";
1924         return first.cs() == "global" && second.cs() == "long" &&
1925                third.cs() == "def";
1926 }
1927
1928
1929 /// Parse a macro definition (assumes that is_macro() returned true)
1930 void parse_macro(Parser & p, ostream & os, Context & context)
1931 {
1932         context.check_layout(os);
1933         Token first = p.curr_token();
1934         Token second;
1935         Token third;
1936         string command = first.asInput();
1937         if (first.cs() != "def") {
1938                 p.get_token();
1939                 eat_whitespace(p, os, context, false);
1940                 second = p.curr_token();
1941                 command += second.asInput();
1942                 if (second.cs() != "def") {
1943                         p.get_token();
1944                         eat_whitespace(p, os, context, false);
1945                         third = p.curr_token();
1946                         command += third.asInput();
1947                 }
1948         }
1949         eat_whitespace(p, os, context, false);
1950         string const name = p.get_token().cs();
1951         eat_whitespace(p, os, context, false);
1952
1953         // parameter text
1954         bool simple = true;
1955         string paramtext;
1956         int arity = 0;
1957         while (p.next_token().cat() != catBegin) {
1958                 if (p.next_token().cat() == catParameter) {
1959                         // # found
1960                         p.get_token();
1961                         paramtext += "#";
1962
1963                         // followed by number?
1964                         if (p.next_token().cat() == catOther) {
1965                                 char c = p.getChar();
1966                                 paramtext += c;
1967                                 // number = current arity + 1?
1968                                 if (c == arity + '0' + 1)
1969                                         ++arity;
1970                                 else
1971                                         simple = false;
1972                         } else
1973                                 paramtext += p.get_token().cs();
1974                 } else {
1975                         paramtext += p.get_token().cs();
1976                         simple = false;
1977                 }
1978         }
1979
1980         // only output simple (i.e. compatible) macro as FormulaMacros
1981         string ert = '\\' + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
1982         if (simple) {
1983                 context.check_layout(os);
1984                 begin_inset(os, "FormulaMacro");
1985                 os << "\n\\def" << ert;
1986                 end_inset(os);
1987         } else
1988                 handle_ert(os, command + ert, context);
1989 }
1990
1991
1992 void registerExternalTemplatePackages(string const & name)
1993 {
1994         external::TemplateManager const & etm = external::TemplateManager::get();
1995         external::Template const * const et = etm.getTemplateByName(name);
1996         if (!et)
1997                 return;
1998         external::Template::Formats::const_iterator cit = et->formats.end();
1999         if (pdflatex)
2000                 cit = et->formats.find("PDFLaTeX");
2001         if (cit == et->formats.end())
2002                 // If the template has not specified a PDFLaTeX output,
2003                 // we try the LaTeX format.
2004                 cit = et->formats.find("LaTeX");
2005         if (cit == et->formats.end())
2006                 return;
2007         vector<string>::const_iterator qit = cit->second.requirements.begin();
2008         vector<string>::const_iterator qend = cit->second.requirements.end();
2009         for (; qit != qend; ++qit)
2010                 preamble.registerAutomaticallyLoadedPackage(*qit);
2011 }
2012
2013 } // anonymous namespace
2014
2015
2016 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
2017                 Context & context)
2018 {
2019         Layout const * newlayout = 0;
2020         InsetLayout const * newinsetlayout = 0;
2021         char const * const * where = 0;
2022         // Store the latest bibliographystyle and nocite{*} option
2023         // (needed for bibtex inset)
2024         string btprint;
2025         string bibliographystyle = "default";
2026         bool const use_natbib = preamble.isPackageUsed("natbib");
2027         bool const use_jurabib = preamble.isPackageUsed("jurabib");
2028         string last_env;
2029         while (p.good()) {
2030                 Token const & t = p.get_token();
2031
2032 #ifdef FILEDEBUG
2033                 debugToken(cerr, t, flags);
2034 #endif
2035
2036                 if (flags & FLAG_ITEM) {
2037                         if (t.cat() == catSpace)
2038                                 continue;
2039
2040                         flags &= ~FLAG_ITEM;
2041                         if (t.cat() == catBegin) {
2042                                 // skip the brace and collect everything to the next matching
2043                                 // closing brace
2044                                 flags |= FLAG_BRACE_LAST;
2045                                 continue;
2046                         }
2047
2048                         // handle only this single token, leave the loop if done
2049                         flags |= FLAG_LEAVE;
2050                 }
2051
2052                 if (t.cat() != catEscape && t.character() == ']' &&
2053                     (flags & FLAG_BRACK_LAST))
2054                         return;
2055                 if (t.cat() == catEnd && (flags & FLAG_BRACE_LAST))
2056                         return;
2057
2058                 // If there is anything between \end{env} and \begin{env} we
2059                 // don't need to output a separator.
2060                 if (t.cat() != catSpace && t.cat() != catNewline &&
2061                     t.asInput() != "\\begin")
2062                         last_env = "";
2063
2064                 //
2065                 // cat codes
2066                 //
2067                 if (t.cat() == catMath) {
2068                         // we are inside some text mode thingy, so opening new math is allowed
2069                         context.check_layout(os);
2070                         begin_inset(os, "Formula ");
2071                         Token const & n = p.get_token();
2072                         bool const display(n.cat() == catMath && outer);
2073                         if (display) {
2074                                 // TeX's $$...$$ syntax for displayed math
2075                                 os << "\\[";
2076                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2077                                 os << "\\]";
2078                                 p.get_token(); // skip the second '$' token
2079                         } else {
2080                                 // simple $...$  stuff
2081                                 p.putback();
2082                                 os << '$';
2083                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2084                                 os << '$';
2085                         }
2086                         end_inset(os);
2087                         if (display) {
2088                                 // Prevent the conversion of a line break to a
2089                                 // space (bug 7668). This does not change the
2090                                 // output, but looks ugly in LyX.
2091                                 eat_whitespace(p, os, context, false);
2092                         }
2093                 }
2094
2095                 else if (t.cat() == catSuper || t.cat() == catSub)
2096                         cerr << "catcode " << t << " illegal in text mode\n";
2097
2098                 // Basic support for english quotes. This should be
2099                 // extended to other quotes, but is not so easy (a
2100                 // left english quote is the same as a right german
2101                 // quote...)
2102                 else if (t.asInput() == "`" && p.next_token().asInput() == "`") {
2103                         context.check_layout(os);
2104                         begin_inset(os, "Quotes ");
2105                         os << "eld";
2106                         end_inset(os);
2107                         p.get_token();
2108                         skip_braces(p);
2109                 }
2110                 else if (t.asInput() == "'" && p.next_token().asInput() == "'") {
2111                         context.check_layout(os);
2112                         begin_inset(os, "Quotes ");
2113                         os << "erd";
2114                         end_inset(os);
2115                         p.get_token();
2116                         skip_braces(p);
2117                 }
2118
2119                 else if (t.asInput() == ">" && p.next_token().asInput() == ">") {
2120                         context.check_layout(os);
2121                         begin_inset(os, "Quotes ");
2122                         os << "ald";
2123                         end_inset(os);
2124                         p.get_token();
2125                         skip_braces(p);
2126                 }
2127
2128                 else if (t.asInput() == "<" && p.next_token().asInput() == "<") {
2129                         context.check_layout(os);
2130                         begin_inset(os, "Quotes ");
2131                         os << "ard";
2132                         end_inset(os);
2133                         p.get_token();
2134                         skip_braces(p);
2135                 }
2136
2137                 else if (t.asInput() == "<"
2138                          && p.next_token().asInput() == "<" && noweb_mode) {
2139                         p.get_token();
2140                         parse_noweb(p, os, context);
2141                 }
2142
2143                 else if (t.cat() == catSpace || (t.cat() == catNewline && ! p.isParagraph()))
2144                         check_space(p, os, context);
2145
2146                 else if (t.character() == '[' && noweb_mode &&
2147                          p.next_token().character() == '[') {
2148                         // These can contain underscores
2149                         p.putback();
2150                         string const s = p.getFullOpt() + ']';
2151                         if (p.next_token().character() == ']')
2152                                 p.get_token();
2153                         else
2154                                 cerr << "Warning: Inserting missing ']' in '"
2155                                      << s << "'." << endl;
2156                         handle_ert(os, s, context);
2157                 }
2158
2159                 else if (t.cat() == catLetter) {
2160                         context.check_layout(os);
2161                         // Workaround for bug 4752.
2162                         // FIXME: This whole code block needs to be removed
2163                         //        when the bug is fixed and tex2lyx produces
2164                         //        the updated file format.
2165                         // The replacement algorithm in LyX is so stupid that
2166                         // it even translates a phrase if it is part of a word.
2167                         bool handled = false;
2168                         for (int const * l = known_phrase_lengths; *l; ++l) {
2169                                 string phrase = t.cs();
2170                                 for (int i = 1; i < *l && p.next_token().isAlnumASCII(); ++i)
2171                                         phrase += p.get_token().cs();
2172                                 if (is_known(phrase, known_coded_phrases)) {
2173                                         handle_ert(os, phrase, context);
2174                                         handled = true;
2175                                         break;
2176                                 } else {
2177                                         for (size_t i = 1; i < phrase.length(); ++i)
2178                                                 p.putback();
2179                                 }
2180                         }
2181                         if (!handled)
2182                                 os << t.cs();
2183                 }
2184
2185                 else if (t.cat() == catOther ||
2186                                t.cat() == catAlign ||
2187                                t.cat() == catParameter) {
2188                         // This translates "&" to "\\&" which may be wrong...
2189                         context.check_layout(os);
2190                         os << t.cs();
2191                 }
2192
2193                 else if (p.isParagraph()) {
2194                         if (context.new_layout_allowed)
2195                                 context.new_paragraph(os);
2196                         else
2197                                 handle_ert(os, "\\par ", context);
2198                         eat_whitespace(p, os, context, true);
2199                 }
2200
2201                 else if (t.cat() == catActive) {
2202                         context.check_layout(os);
2203                         if (t.character() == '~') {
2204                                 if (context.layout->free_spacing)
2205                                         os << ' ';
2206                                 else {
2207                                         begin_inset(os, "space ~\n");
2208                                         end_inset(os);
2209                                 }
2210                         } else
2211                                 os << t.cs();
2212                 }
2213
2214                 else if (t.cat() == catBegin) {
2215                         Token const next = p.next_token();
2216                         Token const end = p.next_next_token();
2217                         if (next.cat() == catEnd) {
2218                         // {}
2219                         Token const prev = p.prev_token();
2220                         p.get_token();
2221                         if (p.next_token().character() == '`' ||
2222                             (prev.character() == '-' &&
2223                              p.next_token().character() == '-'))
2224                                 ; // ignore it in {}`` or -{}-
2225                         else
2226                                 handle_ert(os, "{}", context);
2227                         } else if (next.cat() == catEscape &&
2228                                    is_known(next.cs(), known_quotes) &&
2229                                    end.cat() == catEnd) {
2230                                 // Something like {\textquoteright} (e.g.
2231                                 // from writer2latex). LyX writes
2232                                 // \textquoteright{}, so we may skip the
2233                                 // braces here for better readability.
2234                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2235                                                    outer, context);
2236                         } else {
2237                         context.check_layout(os);
2238                         // special handling of font attribute changes
2239                         Token const prev = p.prev_token();
2240                         TeXFont const oldFont = context.font;
2241                         if (next.character() == '[' ||
2242                             next.character() == ']' ||
2243                             next.character() == '*') {
2244                                 p.get_token();
2245                                 if (p.next_token().cat() == catEnd) {
2246                                         os << next.cs();
2247                                         p.get_token();
2248                                 } else {
2249                                         p.putback();
2250                                         handle_ert(os, "{", context);
2251                                         parse_text_snippet(p, os,
2252                                                         FLAG_BRACE_LAST,
2253                                                         outer, context);
2254                                         handle_ert(os, "}", context);
2255                                 }
2256                         } else if (! context.new_layout_allowed) {
2257                                 handle_ert(os, "{", context);
2258                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2259                                                    outer, context);
2260                                 handle_ert(os, "}", context);
2261                         } else if (is_known(next.cs(), known_sizes)) {
2262                                 // next will change the size, so we must
2263                                 // reset it here
2264                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2265                                                    outer, context);
2266                                 if (!context.atParagraphStart())
2267                                         os << "\n\\size "
2268                                            << context.font.size << "\n";
2269                         } else if (is_known(next.cs(), known_font_families)) {
2270                                 // next will change the font family, so we
2271                                 // must reset it here
2272                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2273                                                    outer, context);
2274                                 if (!context.atParagraphStart())
2275                                         os << "\n\\family "
2276                                            << context.font.family << "\n";
2277                         } else if (is_known(next.cs(), known_font_series)) {
2278                                 // next will change the font series, so we
2279                                 // must reset it here
2280                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2281                                                    outer, context);
2282                                 if (!context.atParagraphStart())
2283                                         os << "\n\\series "
2284                                            << context.font.series << "\n";
2285                         } else if (is_known(next.cs(), known_font_shapes)) {
2286                                 // next will change the font shape, so we
2287                                 // must reset it here
2288                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2289                                                    outer, context);
2290                                 if (!context.atParagraphStart())
2291                                         os << "\n\\shape "
2292                                            << context.font.shape << "\n";
2293                         } else if (is_known(next.cs(), known_old_font_families) ||
2294                                    is_known(next.cs(), known_old_font_series) ||
2295                                    is_known(next.cs(), known_old_font_shapes)) {
2296                                 // next will change the font family, series
2297                                 // and shape, so we must reset it here
2298                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2299                                                    outer, context);
2300                                 if (!context.atParagraphStart())
2301                                         os <<  "\n\\family "
2302                                            << context.font.family
2303                                            << "\n\\series "
2304                                            << context.font.series
2305                                            << "\n\\shape "
2306                                            << context.font.shape << "\n";
2307                         } else {
2308                                 handle_ert(os, "{", context);
2309                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2310                                                    outer, context);
2311                                 handle_ert(os, "}", context);
2312                                 }
2313                         }
2314                 }
2315
2316                 else if (t.cat() == catEnd) {
2317                         if (flags & FLAG_BRACE_LAST) {
2318                                 return;
2319                         }
2320                         cerr << "stray '}' in text\n";
2321                         handle_ert(os, "}", context);
2322                 }
2323
2324                 else if (t.cat() == catComment)
2325                         parse_comment(p, os, t, context);
2326
2327                 //
2328                 // control sequences
2329                 //
2330
2331                 else if (t.cs() == "(") {
2332                         context.check_layout(os);
2333                         begin_inset(os, "Formula");
2334                         os << " \\(";
2335                         parse_math(p, os, FLAG_SIMPLE2, MATH_MODE);
2336                         os << "\\)";
2337                         end_inset(os);
2338                 }
2339
2340                 else if (t.cs() == "[") {
2341                         context.check_layout(os);
2342                         begin_inset(os, "Formula");
2343                         os << " \\[";
2344                         parse_math(p, os, FLAG_EQUATION, MATH_MODE);
2345                         os << "\\]";
2346                         end_inset(os);
2347                         // Prevent the conversion of a line break to a space
2348                         // (bug 7668). This does not change the output, but
2349                         // looks ugly in LyX.
2350                         eat_whitespace(p, os, context, false);
2351                 }
2352
2353                 else if (t.cs() == "begin")
2354                         parse_environment(p, os, outer, last_env,
2355                                           context);
2356
2357                 else if (t.cs() == "end") {
2358                         if (flags & FLAG_END) {
2359                                 // eat environment name
2360                                 string const name = p.getArg('{', '}');
2361                                 if (name != active_environment())
2362                                         cerr << "\\end{" + name + "} does not match \\begin{"
2363                                                 + active_environment() + "}\n";
2364                                 return;
2365                         }
2366                         p.error("found 'end' unexpectedly");
2367                 }
2368
2369                 else if (t.cs() == "item") {
2370                         string s;
2371                         bool const optarg = p.hasOpt();
2372                         if (optarg) {
2373                                 // FIXME: This swallows comments, but we cannot use
2374                                 //        eat_whitespace() since we must not output
2375                                 //        anything before the item.
2376                                 p.skip_spaces(true);
2377                                 s = p.verbatimOption();
2378                         } else
2379                                 p.skip_spaces(false);
2380                         context.set_item();
2381                         context.check_layout(os);
2382                         if (context.has_item) {
2383                                 // An item in an unknown list-like environment
2384                                 // FIXME: Do this in check_layout()!
2385                                 context.has_item = false;
2386                                 if (optarg)
2387                                         handle_ert(os, "\\item", context);
2388                                 else
2389                                         handle_ert(os, "\\item ", context);
2390                         }
2391                         if (optarg) {
2392                                 if (context.layout->labeltype != LABEL_MANUAL) {
2393                                         // LyX does not support \item[\mybullet]
2394                                         // in itemize environments
2395                                         Parser p2(s + ']');
2396                                         os << parse_text_snippet(p2,
2397                                                 FLAG_BRACK_LAST, outer, context);
2398                                 } else if (!s.empty()) {
2399                                         // LyX adds braces around the argument,
2400                                         // so we need to remove them here.
2401                                         if (s.size() > 2 && s[0] == '{' &&
2402                                             s[s.size()-1] == '}')
2403                                                 s = s.substr(1, s.size()-2);
2404                                         // If the argument contains a space we
2405                                         // must put it into ERT: Otherwise LyX
2406                                         // would misinterpret the space as
2407                                         // item delimiter (bug 7663)
2408                                         if (contains(s, ' ')) {
2409                                                 handle_ert(os, s, context);
2410                                         } else {
2411                                                 Parser p2(s + ']');
2412                                                 os << parse_text_snippet(p2,
2413                                                         FLAG_BRACK_LAST,
2414                                                         outer, context);
2415                                         }
2416                                         // The space is needed to separate the
2417                                         // item from the rest of the sentence.
2418                                         os << ' ';
2419                                         eat_whitespace(p, os, context, false);
2420                                 }
2421                         }
2422                 }
2423
2424                 else if (t.cs() == "bibitem") {
2425                         context.set_item();
2426                         context.check_layout(os);
2427                         eat_whitespace(p, os, context, false);
2428                         string label = convert_command_inset_arg(p.verbatimOption());
2429                         string key = convert_command_inset_arg(p.verbatim_item());
2430                         if (contains(label, '\\') || contains(key, '\\')) {
2431                                 // LyX can't handle LaTeX commands in labels or keys
2432                                 handle_ert(os, t.asInput() + '[' + label +
2433                                                "]{" + p.verbatim_item() + '}',
2434                                            context);
2435                         } else {
2436                                 begin_command_inset(os, "bibitem", "bibitem");
2437                                 os << "label \"" << label << "\"\n"
2438                                       "key \"" << key << "\"\n";
2439                                 end_inset(os);
2440                         }
2441                 }
2442
2443                 else if (is_macro(p)) {
2444                         // catch the case of \def\inputGnumericTable
2445                         bool macro = true;
2446                         if (t.cs() == "def") {
2447                                 Token second = p.next_token();
2448                                 if (second.cs() == "inputGnumericTable") {
2449                                         p.pushPosition();
2450                                         p.get_token();
2451                                         skip_braces(p);
2452                                         Token third = p.get_token();
2453                                         p.popPosition();
2454                                         if (third.cs() == "input") {
2455                                                 p.get_token();
2456                                                 skip_braces(p);
2457                                                 p.get_token();
2458                                                 string name = normalize_filename(p.verbatim_item());
2459                                                 string const path = getMasterFilePath();
2460                                                 // We want to preserve relative / absolute filenames,
2461                                                 // therefore path is only used for testing
2462                                                 // The file extension is in every case ".tex".
2463                                                 // So we need to remove this extension and check for
2464                                                 // the original one.
2465                                                 name = removeExtension(name);
2466                                                 if (!makeAbsPath(name, path).exists()) {
2467                                                         char const * const Gnumeric_formats[] = {"gnumeric",
2468                                                                 "ods", "xls", 0};
2469                                                         string const Gnumeric_name =
2470                                                                 find_file(name, path, Gnumeric_formats);
2471                                                         if (!Gnumeric_name.empty())
2472                                                                 name = Gnumeric_name;
2473                                                 }
2474                                                 if (makeAbsPath(name, path).exists())
2475                                                         fix_relative_filename(name);
2476                                                 else
2477                                                         cerr << "Warning: Could not find file '"
2478                                                              << name << "'." << endl;
2479                                                 context.check_layout(os);
2480                                                 begin_inset(os, "External\n\ttemplate ");
2481                                                 os << "GnumericSpreadsheet\n\tfilename "
2482                                                    << name << "\n";
2483                                                 end_inset(os);
2484                                                 context.check_layout(os);
2485                                                 macro = false;
2486                                                 // register the packages that are automatically reloaded
2487                                                 // by the Gnumeric template
2488                                                 registerExternalTemplatePackages("GnumericSpreadsheet");
2489                                         }
2490                                 }
2491                         }
2492                         if (macro)
2493                                 parse_macro(p, os, context);
2494                 }
2495
2496                 else if (t.cs() == "noindent") {
2497                         p.skip_spaces();
2498                         context.add_par_extra_stuff("\\noindent\n");
2499                 }
2500
2501                 else if (t.cs() == "appendix") {
2502                         context.add_par_extra_stuff("\\start_of_appendix\n");
2503                         // We need to start a new paragraph. Otherwise the
2504                         // appendix in 'bla\appendix\chapter{' would start
2505                         // too late.
2506                         context.new_paragraph(os);
2507                         // We need to make sure that the paragraph is
2508                         // generated even if it is empty. Otherwise the
2509                         // appendix in '\par\appendix\par\chapter{' would
2510                         // start too late.
2511                         context.check_layout(os);
2512                         // FIXME: This is a hack to prevent paragraph
2513                         // deletion if it is empty. Handle this better!
2514                         handle_comment(os,
2515                                 "%dummy comment inserted by tex2lyx to "
2516                                 "ensure that this paragraph is not empty",
2517                                 context);
2518                         // Both measures above may generate an additional
2519                         // empty paragraph, but that does not hurt, because
2520                         // whitespace does not matter here.
2521                         eat_whitespace(p, os, context, true);
2522                 }
2523
2524                 // Must catch empty dates before findLayout is called below
2525                 else if (t.cs() == "date") {
2526                         eat_whitespace(p, os, context, false);
2527                         p.pushPosition();
2528                         string const date = p.verbatim_item();
2529                         p.popPosition();
2530                         if (date.empty()) {
2531                                 preamble.suppressDate(true);
2532                                 p.verbatim_item();
2533                         } else {
2534                                 preamble.suppressDate(false);
2535                                 if (context.new_layout_allowed &&
2536                                     (newlayout = findLayout(context.textclass,
2537                                                             t.cs(), true))) {
2538                                         // write the layout
2539                                         output_command_layout(os, p, outer,
2540                                                         context, newlayout);
2541                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2542                                         if (!preamble.titleLayoutFound())
2543                                                 preamble.titleLayoutFound(newlayout->intitle);
2544                                         set<string> const & req = newlayout->requires();
2545                                         set<string>::const_iterator it = req.begin();
2546                                         set<string>::const_iterator en = req.end();
2547                                         for (; it != en; ++it)
2548                                                 preamble.registerAutomaticallyLoadedPackage(*it);
2549                                 } else
2550                                         handle_ert(os,
2551                                                 "\\date{" + p.verbatim_item() + '}',
2552                                                 context);
2553                         }
2554                 }
2555
2556                 // Starred section headings
2557                 // Must attempt to parse "Section*" before "Section".
2558                 else if ((p.next_token().asInput() == "*") &&
2559                          context.new_layout_allowed &&
2560                          (newlayout = findLayout(context.textclass, t.cs() + '*', true))) {
2561                         // write the layout
2562                         p.get_token();
2563                         output_command_layout(os, p, outer, context, newlayout);
2564                         p.skip_spaces();
2565                         if (!preamble.titleLayoutFound())
2566                                 preamble.titleLayoutFound(newlayout->intitle);
2567                         set<string> const & req = newlayout->requires();
2568                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2569                                 preamble.registerAutomaticallyLoadedPackage(*it);
2570                 }
2571
2572                 // Section headings and the like
2573                 else if (context.new_layout_allowed &&
2574                          (newlayout = findLayout(context.textclass, t.cs(), true))) {
2575                         // write the layout
2576                         output_command_layout(os, p, outer, context, newlayout);
2577                         p.skip_spaces();
2578                         if (!preamble.titleLayoutFound())
2579                                 preamble.titleLayoutFound(newlayout->intitle);
2580                         set<string> const & req = newlayout->requires();
2581                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2582                                 preamble.registerAutomaticallyLoadedPackage(*it);
2583                 }
2584
2585                 else if (t.cs() == "caption") {
2586                         p.skip_spaces();
2587                         context.check_layout(os);
2588                         p.skip_spaces();
2589                         begin_inset(os, "Caption\n");
2590                         Context newcontext(true, context.textclass);
2591                         newcontext.font = context.font;
2592                         newcontext.check_layout(os);
2593                         if (p.next_token().cat() != catEscape &&
2594                             p.next_token().character() == '[') {
2595                                 p.get_token(); // eat '['
2596                                 begin_inset(os, "Argument\n");
2597                                 os << "status collapsed\n";
2598                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
2599                                 end_inset(os);
2600                                 eat_whitespace(p, os, context, false);
2601                         }
2602                         parse_text(p, os, FLAG_ITEM, outer, context);
2603                         context.check_end_layout(os);
2604                         // We don't need really a new paragraph, but
2605                         // we must make sure that the next item gets a \begin_layout.
2606                         context.new_paragraph(os);
2607                         end_inset(os);
2608                         p.skip_spaces();
2609                         newcontext.check_end_layout(os);
2610                 }
2611
2612                 else if (t.cs() == "subfloat") {
2613                         // the syntax is \subfloat[caption]{content}
2614                         // if it is a table of figure depends on the surrounding float
2615                         bool has_caption = false;
2616                         p.skip_spaces();
2617                         // do nothing if there is no outer float
2618                         if (!float_type.empty()) {
2619                                 context.check_layout(os);
2620                                 p.skip_spaces();
2621                                 begin_inset(os, "Float " + float_type + "\n");
2622                                 os << "wide false"
2623                                    << "\nsideways false"
2624                                    << "\nstatus collapsed\n\n";
2625                                 // test for caption
2626                                 string caption;
2627                                 if (p.next_token().cat() != catEscape &&
2628                                                 p.next_token().character() == '[') {
2629                                                         p.get_token(); // eat '['
2630                                                         caption = parse_text_snippet(p, FLAG_BRACK_LAST, outer, context);
2631                                                         has_caption = true;
2632                                 }
2633                                 // the content
2634                                 parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
2635                                 // the caption comes always as the last
2636                                 if (has_caption) {
2637                                         // we must make sure that the caption gets a \begin_layout
2638                                         os << "\n\\begin_layout Plain Layout";
2639                                         p.skip_spaces();
2640                                         begin_inset(os, "Caption\n");
2641                                         Context newcontext(true, context.textclass);
2642                                         newcontext.font = context.font;
2643                                         newcontext.check_layout(os);
2644                                         os << caption << "\n";
2645                                         newcontext.check_end_layout(os);
2646                                         // We don't need really a new paragraph, but
2647                                         // we must make sure that the next item gets a \begin_layout.
2648                                         //newcontext.new_paragraph(os);
2649                                         end_inset(os);
2650                                         p.skip_spaces();
2651                                 }
2652                                 // We don't need really a new paragraph, but
2653                                 // we must make sure that the next item gets a \begin_layout.
2654                                 if (has_caption)
2655                                         context.new_paragraph(os);
2656                                 end_inset(os);
2657                                 p.skip_spaces();
2658                                 context.check_end_layout(os);
2659                                 // close the layout we opened
2660                                 if (has_caption)
2661                                         os << "\n\\end_layout\n";
2662                         } else {
2663                                 // if the float type is not supported or there is no surrounding float
2664                                 // output it as ERT
2665                                 if (p.hasOpt()) {
2666                                         string opt_arg = convert_command_inset_arg(p.getArg('[', ']'));
2667                                         handle_ert(os, t.asInput() + '[' + opt_arg +
2668                                                "]{" + p.verbatim_item() + '}', context);
2669                                 } else
2670                                         handle_ert(os, t.asInput() + "{" + p.verbatim_item() + '}', context);
2671                         }
2672                 }
2673
2674                 else if (t.cs() == "includegraphics") {
2675                         bool const clip = p.next_token().asInput() == "*";
2676                         if (clip)
2677                                 p.get_token();
2678                         string const arg = p.getArg('[', ']');
2679                         map<string, string> opts;
2680                         vector<string> keys;
2681                         split_map(arg, opts, keys);
2682                         if (clip)
2683                                 opts["clip"] = string();
2684                         string name = normalize_filename(p.verbatim_item());
2685
2686                         string const path = getMasterFilePath();
2687                         // We want to preserve relative / absolute filenames,
2688                         // therefore path is only used for testing
2689                         if (!makeAbsPath(name, path).exists()) {
2690                                 // The file extension is probably missing.
2691                                 // Now try to find it out.
2692                                 string const dvips_name =
2693                                         find_file(name, path,
2694                                                   known_dvips_graphics_formats);
2695                                 string const pdftex_name =
2696                                         find_file(name, path,
2697                                                   known_pdftex_graphics_formats);
2698                                 if (!dvips_name.empty()) {
2699                                         if (!pdftex_name.empty()) {
2700                                                 cerr << "This file contains the "
2701                                                         "latex snippet\n"
2702                                                         "\"\\includegraphics{"
2703                                                      << name << "}\".\n"
2704                                                         "However, files\n\""
2705                                                      << dvips_name << "\" and\n\""
2706                                                      << pdftex_name << "\"\n"
2707                                                         "both exist, so I had to make a "
2708                                                         "choice and took the first one.\n"
2709                                                         "Please move the unwanted one "
2710                                                         "someplace else and try again\n"
2711                                                         "if my choice was wrong."
2712                                                      << endl;
2713                                         }
2714                                         name = dvips_name;
2715                                 } else if (!pdftex_name.empty()) {
2716                                         name = pdftex_name;
2717                                         pdflatex = true;
2718                                 }
2719                         }
2720
2721                         if (makeAbsPath(name, path).exists())
2722                                 fix_relative_filename(name);
2723                         else
2724                                 cerr << "Warning: Could not find graphics file '"
2725                                      << name << "'." << endl;
2726
2727                         context.check_layout(os);
2728                         begin_inset(os, "Graphics ");
2729                         os << "\n\tfilename " << name << '\n';
2730                         if (opts.find("width") != opts.end())
2731                                 os << "\twidth "
2732                                    << translate_len(opts["width"]) << '\n';
2733                         if (opts.find("height") != opts.end())
2734                                 os << "\theight "
2735                                    << translate_len(opts["height"]) << '\n';
2736                         if (opts.find("scale") != opts.end()) {
2737                                 istringstream iss(opts["scale"]);
2738                                 double val;
2739                                 iss >> val;
2740                                 val = val*100;
2741                                 os << "\tscale " << val << '\n';
2742                         }
2743                         if (opts.find("angle") != opts.end()) {
2744                                 os << "\trotateAngle "
2745                                    << opts["angle"] << '\n';
2746                                 vector<string>::const_iterator a =
2747                                         find(keys.begin(), keys.end(), "angle");
2748                                 vector<string>::const_iterator s =
2749                                         find(keys.begin(), keys.end(), "width");
2750                                 if (s == keys.end())
2751                                         s = find(keys.begin(), keys.end(), "height");
2752                                 if (s == keys.end())
2753                                         s = find(keys.begin(), keys.end(), "scale");
2754                                 if (s != keys.end() && distance(s, a) > 0)
2755                                         os << "\tscaleBeforeRotation\n";
2756                         }
2757                         if (opts.find("origin") != opts.end()) {
2758                                 ostringstream ss;
2759                                 string const opt = opts["origin"];
2760                                 if (opt.find('l') != string::npos) ss << "left";
2761                                 if (opt.find('r') != string::npos) ss << "right";
2762                                 if (opt.find('c') != string::npos) ss << "center";
2763                                 if (opt.find('t') != string::npos) ss << "Top";
2764                                 if (opt.find('b') != string::npos) ss << "Bottom";
2765                                 if (opt.find('B') != string::npos) ss << "Baseline";
2766                                 if (!ss.str().empty())
2767                                         os << "\trotateOrigin " << ss.str() << '\n';
2768                                 else
2769                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
2770                         }
2771                         if (opts.find("keepaspectratio") != opts.end())
2772                                 os << "\tkeepAspectRatio\n";
2773                         if (opts.find("clip") != opts.end())
2774                                 os << "\tclip\n";
2775                         if (opts.find("draft") != opts.end())
2776                                 os << "\tdraft\n";
2777                         if (opts.find("bb") != opts.end())
2778                                 os << "\tBoundingBox "
2779                                    << opts["bb"] << '\n';
2780                         int numberOfbbOptions = 0;
2781                         if (opts.find("bbllx") != opts.end())
2782                                 numberOfbbOptions++;
2783                         if (opts.find("bblly") != opts.end())
2784                                 numberOfbbOptions++;
2785                         if (opts.find("bburx") != opts.end())
2786                                 numberOfbbOptions++;
2787                         if (opts.find("bbury") != opts.end())
2788                                 numberOfbbOptions++;
2789                         if (numberOfbbOptions == 4)
2790                                 os << "\tBoundingBox "
2791                                    << opts["bbllx"] << " " << opts["bblly"] << " "
2792                                    << opts["bburx"] << " " << opts["bbury"] << '\n';
2793                         else if (numberOfbbOptions > 0)
2794                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2795                         numberOfbbOptions = 0;
2796                         if (opts.find("natwidth") != opts.end())
2797                                 numberOfbbOptions++;
2798                         if (opts.find("natheight") != opts.end())
2799                                 numberOfbbOptions++;
2800                         if (numberOfbbOptions == 2)
2801                                 os << "\tBoundingBox 0bp 0bp "
2802                                    << opts["natwidth"] << " " << opts["natheight"] << '\n';
2803                         else if (numberOfbbOptions > 0)
2804                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2805                         ostringstream special;
2806                         if (opts.find("hiresbb") != opts.end())
2807                                 special << "hiresbb,";
2808                         if (opts.find("trim") != opts.end())
2809                                 special << "trim,";
2810                         if (opts.find("viewport") != opts.end())
2811                                 special << "viewport=" << opts["viewport"] << ',';
2812                         if (opts.find("totalheight") != opts.end())
2813                                 special << "totalheight=" << opts["totalheight"] << ',';
2814                         if (opts.find("type") != opts.end())
2815                                 special << "type=" << opts["type"] << ',';
2816                         if (opts.find("ext") != opts.end())
2817                                 special << "ext=" << opts["ext"] << ',';
2818                         if (opts.find("read") != opts.end())
2819                                 special << "read=" << opts["read"] << ',';
2820                         if (opts.find("command") != opts.end())
2821                                 special << "command=" << opts["command"] << ',';
2822                         string s_special = special.str();
2823                         if (!s_special.empty()) {
2824                                 // We had special arguments. Remove the trailing ','.
2825                                 os << "\tspecial " << s_special.substr(0, s_special.size() - 1) << '\n';
2826                         }
2827                         // TODO: Handle the unknown settings better.
2828                         // Warn about invalid options.
2829                         // Check whether some option was given twice.
2830                         end_inset(os);
2831                         preamble.registerAutomaticallyLoadedPackage("graphicx");
2832                 }
2833
2834                 else if (t.cs() == "footnote" ||
2835                          (t.cs() == "thanks" && context.layout->intitle)) {
2836                         p.skip_spaces();
2837                         context.check_layout(os);
2838                         begin_inset(os, "Foot\n");
2839                         os << "status collapsed\n\n";
2840                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2841                         end_inset(os);
2842                 }
2843
2844                 else if (t.cs() == "marginpar") {
2845                         p.skip_spaces();
2846                         context.check_layout(os);
2847                         begin_inset(os, "Marginal\n");
2848                         os << "status collapsed\n\n";
2849                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2850                         end_inset(os);
2851                 }
2852
2853                 else if (t.cs() == "lstinline") {
2854                         p.skip_spaces();
2855                         parse_listings(p, os, context, true);
2856                 }
2857
2858                 else if (t.cs() == "ensuremath") {
2859                         p.skip_spaces();
2860                         context.check_layout(os);
2861                         string const s = p.verbatim_item();
2862                         //FIXME: this never triggers in UTF8
2863                         if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5")
2864                                 os << s;
2865                         else
2866                                 handle_ert(os, "\\ensuremath{" + s + "}",
2867                                            context);
2868                 }
2869
2870                 else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
2871                         if (preamble.titleLayoutFound()) {
2872                                 // swallow this
2873                                 skip_spaces_braces(p);
2874                         } else
2875                                 handle_ert(os, t.asInput(), context);
2876                 }
2877
2878                 else if (t.cs() == "tableofcontents" || t.cs() == "lstlistoflistings") {
2879                         context.check_layout(os);
2880                         begin_command_inset(os, "toc", t.cs());
2881                         end_inset(os);
2882                         skip_spaces_braces(p);
2883                         if (t.cs() == "lstlistoflistings")
2884                                 preamble.registerAutomaticallyLoadedPackage("listings");
2885                 }
2886
2887                 else if (t.cs() == "listoffigures") {
2888                         context.check_layout(os);
2889                         begin_inset(os, "FloatList figure\n");
2890                         end_inset(os);
2891                         skip_spaces_braces(p);
2892                 }
2893
2894                 else if (t.cs() == "listoftables") {
2895                         context.check_layout(os);
2896                         begin_inset(os, "FloatList table\n");
2897                         end_inset(os);
2898                         skip_spaces_braces(p);
2899                 }
2900
2901                 else if (t.cs() == "listof") {
2902                         p.skip_spaces(true);
2903                         string const name = p.get_token().cs();
2904                         if (context.textclass.floats().typeExist(name)) {
2905                                 context.check_layout(os);
2906                                 begin_inset(os, "FloatList ");
2907                                 os << name << "\n";
2908                                 end_inset(os);
2909                                 p.get_token(); // swallow second arg
2910                         } else
2911                                 handle_ert(os, "\\listof{" + name + "}", context);
2912                 }
2913
2914                 else if ((where = is_known(t.cs(), known_text_font_families)))
2915                         parse_text_attributes(p, os, FLAG_ITEM, outer,
2916                                 context, "\\family", context.font.family,
2917                                 known_coded_font_families[where - known_text_font_families]);
2918
2919                 else if ((where = is_known(t.cs(), known_text_font_series)))
2920                         parse_text_attributes(p, os, FLAG_ITEM, outer,
2921                                 context, "\\series", context.font.series,
2922                                 known_coded_font_series[where - known_text_font_series]);
2923
2924                 else if ((where = is_known(t.cs(), known_text_font_shapes)))
2925                         parse_text_attributes(p, os, FLAG_ITEM, outer,
2926                                 context, "\\shape", context.font.shape,
2927                                 known_coded_font_shapes[where - known_text_font_shapes]);
2928
2929                 else if (t.cs() == "textnormal" || t.cs() == "normalfont") {
2930                         context.check_layout(os);
2931                         TeXFont oldFont = context.font;
2932                         context.font.init();
2933                         context.font.size = oldFont.size;
2934                         os << "\n\\family " << context.font.family << "\n";
2935                         os << "\n\\series " << context.font.series << "\n";
2936                         os << "\n\\shape " << context.font.shape << "\n";
2937                         if (t.cs() == "textnormal") {
2938                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2939                                 output_font_change(os, context.font, oldFont);
2940                                 context.font = oldFont;
2941                         } else
2942                                 eat_whitespace(p, os, context, false);
2943                 }
2944
2945                 else if (t.cs() == "textcolor") {
2946                         // scheme is \textcolor{color name}{text}
2947                         string const color = p.verbatim_item();
2948                         // we only support the predefined colors of the color package
2949                         if (color == "black" || color == "blue" || color == "cyan"
2950                                 || color == "green" || color == "magenta" || color == "red"
2951                                 || color == "white" || color == "yellow") {
2952                                         context.check_layout(os);
2953                                         os << "\n\\color " << color << "\n";
2954                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2955                                         context.check_layout(os);
2956                                         os << "\n\\color inherit\n";
2957                                         preamble.registerAutomaticallyLoadedPackage("color");
2958                         } else
2959                                 // for custom defined colors
2960                                 handle_ert(os, t.asInput() + "{" + color + "}", context);
2961                 }
2962
2963                 else if (t.cs() == "underbar" || t.cs() == "uline") {
2964                         // \underbar is not 100% correct (LyX outputs \uline
2965                         // of ulem.sty). The difference is that \ulem allows
2966                         // line breaks, and \underbar does not.
2967                         // Do NOT handle \underline.
2968                         // \underbar cuts through y, g, q, p etc.,
2969                         // \underline does not.
2970                         context.check_layout(os);
2971                         os << "\n\\bar under\n";
2972                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2973                         context.check_layout(os);
2974                         os << "\n\\bar default\n";
2975                         preamble.registerAutomaticallyLoadedPackage("ulem");
2976                 }
2977
2978                 else if (t.cs() == "sout") {
2979                         context.check_layout(os);
2980                         os << "\n\\strikeout on\n";
2981                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2982                         context.check_layout(os);
2983                         os << "\n\\strikeout default\n";
2984                         preamble.registerAutomaticallyLoadedPackage("ulem");
2985                 }
2986
2987                 else if (t.cs() == "uuline" || t.cs() == "uwave" ||
2988                          t.cs() == "emph" || t.cs() == "noun") {
2989                         context.check_layout(os);
2990                         os << "\n\\" << t.cs() << " on\n";
2991                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2992                         context.check_layout(os);
2993                         os << "\n\\" << t.cs() << " default\n";
2994                         if (t.cs() == "uuline" || t.cs() == "uwave")
2995                                 preamble.registerAutomaticallyLoadedPackage("ulem");
2996                 }
2997
2998                 else if (t.cs() == "lyxadded" || t.cs() == "lyxdeleted") {
2999                         context.check_layout(os);
3000                         string name = p.getArg('{', '}');
3001                         string localtime = p.getArg('{', '}');
3002                         preamble.registerAuthor(name);
3003                         Author const & author = preamble.getAuthor(name);
3004                         // from_ctime() will fail if LyX decides to output the
3005                         // time in the text language. It might also use a wrong
3006                         // time zone (if the original LyX document was exported
3007                         // with a different time zone).
3008                         time_t ptime = from_ctime(localtime);
3009                         if (ptime == static_cast<time_t>(-1)) {
3010                                 cerr << "Warning: Could not parse time `" << localtime
3011                                      << "´ for change tracking, using current time instead.\n";
3012                                 ptime = current_time();
3013                         }
3014                         if (t.cs() == "lyxadded")
3015                                 os << "\n\\change_inserted ";
3016                         else
3017                                 os << "\n\\change_deleted ";
3018                         os << author.bufferId() << ' ' << ptime << '\n';
3019                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3020                         bool dvipost    = LaTeXPackages::isAvailable("dvipost");
3021                         bool xcolorulem = LaTeXPackages::isAvailable("ulem") &&
3022                                           LaTeXPackages::isAvailable("xcolor");
3023                         // No need to test for luatex, since luatex comes in
3024                         // two flavours (dvi and pdf), like latex, and those
3025                         // are detected by pdflatex.
3026                         if (pdflatex || xetex) {
3027                                 if (xcolorulem) {
3028                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3029                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3030                                         preamble.registerAutomaticallyLoadedPackage("pdfcolmk");
3031                                 }
3032                         } else {
3033                                 if (dvipost) {
3034                                         preamble.registerAutomaticallyLoadedPackage("dvipost");
3035                                 } else if (xcolorulem) {
3036                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3037                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3038                                 }
3039                         }
3040                 }
3041
3042                 else if (t.cs() == "phantom" || t.cs() == "hphantom" ||
3043                              t.cs() == "vphantom") {
3044                         context.check_layout(os);
3045                         if (t.cs() == "phantom")
3046                                 begin_inset(os, "Phantom Phantom\n");
3047                         if (t.cs() == "hphantom")
3048                                 begin_inset(os, "Phantom HPhantom\n");
3049                         if (t.cs() == "vphantom")
3050                                 begin_inset(os, "Phantom VPhantom\n");
3051                         os << "status open\n";
3052                         parse_text_in_inset(p, os, FLAG_ITEM, outer, context,
3053                                             "Phantom");
3054                         end_inset(os);
3055                 }
3056
3057                 else if (t.cs() == "href") {
3058                         context.check_layout(os);
3059                         string target = p.getArg('{', '}');
3060                         string name = p.getArg('{', '}');
3061                         string type;
3062                         size_t i = target.find(':');
3063                         if (i != string::npos) {
3064                                 type = target.substr(0, i + 1);
3065                                 if (type == "mailto:" || type == "file:")
3066                                         target = target.substr(i + 1);
3067                                 // handle the case that name is equal to target, except of "http://"
3068                                 else if (target.substr(i + 3) == name && type == "http:")
3069                                         target = name;
3070                         }
3071                         begin_command_inset(os, "href", "href");
3072                         if (name != target)
3073                                 os << "name \"" << name << "\"\n";
3074                         os << "target \"" << target << "\"\n";
3075                         if (type == "mailto:" || type == "file:")
3076                                 os << "type \"" << type << "\"\n";
3077                         end_inset(os);
3078                         skip_spaces_braces(p);
3079                 }
3080
3081                 else if (t.cs() == "lyxline") {
3082                         // swallow size argument (it is not used anyway)
3083                         p.getArg('{', '}');
3084                         if (!context.atParagraphStart()) {
3085                                 // so our line is in the middle of a paragraph
3086                                 // we need to add a new line, lest this line
3087                                 // follow the other content on that line and
3088                                 // run off the side of the page
3089                                 // FIXME: This may create an empty paragraph,
3090                                 //        but without that it would not be
3091                                 //        possible to set noindent below.
3092                                 //        Fortunately LaTeX does not care
3093                                 //        about the empty paragraph.
3094                                 context.new_paragraph(os);
3095                         }
3096                         if (preamble.indentParagraphs()) {
3097                                 // we need to unindent, lest the line be too long
3098                                 context.add_par_extra_stuff("\\noindent\n");
3099                         }
3100                         context.check_layout(os);
3101                         begin_command_inset(os, "line", "rule");
3102                         os << "offset \"0.5ex\"\n"
3103                               "width \"100line%\"\n"
3104                               "height \"1pt\"\n";
3105                         end_inset(os);
3106                 }
3107
3108                 else if (t.cs() == "rule") {
3109                         string const offset = (p.hasOpt() ? p.getArg('[', ']') : string());
3110                         string const width = p.getArg('{', '}');
3111                         string const thickness = p.getArg('{', '}');
3112                         context.check_layout(os);
3113                         begin_command_inset(os, "line", "rule");
3114                         if (!offset.empty())
3115                                 os << "offset \"" << translate_len(offset) << "\"\n";
3116                         os << "width \"" << translate_len(width) << "\"\n"
3117                                   "height \"" << translate_len(thickness) << "\"\n";
3118                         end_inset(os);
3119                 }
3120
3121                 else if (is_known(t.cs(), known_phrases) ||
3122                          (t.cs() == "protect" &&
3123                           p.next_token().cat() == catEscape &&
3124                           is_known(p.next_token().cs(), known_phrases))) {
3125                         // LyX sometimes puts a \protect in front, so we have to ignore it
3126                         // FIXME: This needs to be changed when bug 4752 is fixed.
3127                         where = is_known(
3128                                 t.cs() == "protect" ? p.get_token().cs() : t.cs(),
3129                                 known_phrases);
3130                         context.check_layout(os);
3131                         os << known_coded_phrases[where - known_phrases];
3132                         skip_spaces_braces(p);
3133                 }
3134
3135                 else if ((where = is_known(t.cs(), known_ref_commands))) {
3136                         string const opt = p.getOpt();
3137                         if (opt.empty()) {
3138                                 context.check_layout(os);
3139                                 begin_command_inset(os, "ref",
3140                                         known_coded_ref_commands[where - known_ref_commands]);
3141                                 os << "reference \""
3142                                    << convert_command_inset_arg(p.verbatim_item())
3143                                    << "\"\n";
3144                                 end_inset(os);
3145                                 if (t.cs() == "vref" || t.cs() == "vpageref")
3146                                         preamble.registerAutomaticallyLoadedPackage("varioref");
3147
3148                         } else {
3149                                 // LyX does not support optional arguments of ref commands
3150                                 handle_ert(os, t.asInput() + '[' + opt + "]{" +
3151                                                p.verbatim_item() + "}", context);
3152                         }
3153                 }
3154
3155                 else if (use_natbib &&
3156                          is_known(t.cs(), known_natbib_commands) &&
3157                          ((t.cs() != "citefullauthor" &&
3158                            t.cs() != "citeyear" &&
3159                            t.cs() != "citeyearpar") ||
3160                           p.next_token().asInput() != "*")) {
3161                         context.check_layout(os);
3162                         string command = t.cs();
3163                         if (p.next_token().asInput() == "*") {
3164                                 command += '*';
3165                                 p.get_token();
3166                         }
3167                         if (command == "citefullauthor")
3168                                 // alternative name for "\\citeauthor*"
3169                                 command = "citeauthor*";
3170
3171                         // text before the citation
3172                         string before;
3173                         // text after the citation
3174                         string after;
3175                         get_cite_arguments(p, true, before, after);
3176
3177                         if (command == "cite") {
3178                                 // \cite without optional argument means
3179                                 // \citet, \cite with at least one optional
3180                                 // argument means \citep.
3181                                 if (before.empty() && after.empty())
3182                                         command = "citet";
3183                                 else
3184                                         command = "citep";
3185                         }
3186                         if (before.empty() && after == "[]")
3187                                 // avoid \citet[]{a}
3188                                 after.erase();
3189                         else if (before == "[]" && after == "[]") {
3190                                 // avoid \citet[][]{a}
3191                                 before.erase();
3192                                 after.erase();
3193                         }
3194                         // remove the brackets around after and before
3195                         if (!after.empty()) {
3196                                 after.erase(0, 1);
3197                                 after.erase(after.length() - 1, 1);
3198                                 after = convert_command_inset_arg(after);
3199                         }
3200                         if (!before.empty()) {
3201                                 before.erase(0, 1);
3202                                 before.erase(before.length() - 1, 1);
3203                                 before = convert_command_inset_arg(before);
3204                         }
3205                         begin_command_inset(os, "citation", command);
3206                         os << "after " << '"' << after << '"' << "\n";
3207                         os << "before " << '"' << before << '"' << "\n";
3208                         os << "key \""
3209                            << convert_command_inset_arg(p.verbatim_item())
3210                            << "\"\n";
3211                         end_inset(os);
3212                 }
3213
3214                 else if (use_jurabib &&
3215                          is_known(t.cs(), known_jurabib_commands) &&
3216                          (t.cs() == "cite" || p.next_token().asInput() != "*")) {
3217                         context.check_layout(os);
3218                         string command = t.cs();
3219                         if (p.next_token().asInput() == "*") {
3220                                 command += '*';
3221                                 p.get_token();
3222                         }
3223                         char argumentOrder = '\0';
3224                         vector<string> const options =
3225                                 preamble.getPackageOptions("jurabib");
3226                         if (find(options.begin(), options.end(),
3227                                       "natbiborder") != options.end())
3228                                 argumentOrder = 'n';
3229                         else if (find(options.begin(), options.end(),
3230                                            "jurabiborder") != options.end())
3231                                 argumentOrder = 'j';
3232
3233                         // text before the citation
3234                         string before;
3235                         // text after the citation
3236                         string after;
3237                         get_cite_arguments(p, argumentOrder != 'j', before, after);
3238
3239                         string const citation = p.verbatim_item();
3240                         if (!before.empty() && argumentOrder == '\0') {
3241                                 cerr << "Warning: Assuming argument order "
3242                                         "of jurabib version 0.6 for\n'"
3243                                      << command << before << after << '{'
3244                                      << citation << "}'.\n"
3245                                         "Add 'jurabiborder' to the jurabib "
3246                                         "package options if you used an\n"
3247                                         "earlier jurabib version." << endl;
3248                         }
3249                         if (!after.empty()) {
3250                                 after.erase(0, 1);
3251                                 after.erase(after.length() - 1, 1);
3252                         }
3253                         if (!before.empty()) {
3254                                 before.erase(0, 1);
3255                                 before.erase(before.length() - 1, 1);
3256                         }
3257                         begin_command_inset(os, "citation", command);
3258                         os << "after " << '"' << after << '"' << "\n";
3259                         os << "before " << '"' << before << '"' << "\n";
3260                         os << "key " << '"' << citation << '"' << "\n";
3261                         end_inset(os);
3262                 }
3263
3264                 else if (t.cs() == "cite"
3265                         || t.cs() == "nocite") {
3266                         context.check_layout(os);
3267                         string after = convert_command_inset_arg(p.getArg('[', ']'));
3268                         string key = convert_command_inset_arg(p.verbatim_item());
3269                         // store the case that it is "\nocite{*}" to use it later for
3270                         // the BibTeX inset
3271                         if (key != "*") {
3272                                 begin_command_inset(os, "citation", t.cs());
3273                                 os << "after " << '"' << after << '"' << "\n";
3274                                 os << "key " << '"' << key << '"' << "\n";
3275                                 end_inset(os);
3276                         } else if (t.cs() == "nocite")
3277                                 btprint = key;
3278                 }
3279
3280                 else if (t.cs() == "index" ||
3281                          (t.cs() == "sindex" && preamble.use_indices() == "true")) {
3282                         context.check_layout(os);
3283                         string const arg = (t.cs() == "sindex" && p.hasOpt()) ?
3284                                 p.getArg('[', ']') : "";
3285                         string const kind = arg.empty() ? "idx" : arg;
3286                         begin_inset(os, "Index ");
3287                         os << kind << "\nstatus collapsed\n";
3288                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, "Index");
3289                         end_inset(os);
3290                         if (kind != "idx")
3291                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3292                 }
3293
3294                 else if (t.cs() == "nomenclature") {
3295                         context.check_layout(os);
3296                         begin_command_inset(os, "nomenclature", "nomenclature");
3297                         string prefix = convert_command_inset_arg(p.getArg('[', ']'));
3298                         if (!prefix.empty())
3299                                 os << "prefix " << '"' << prefix << '"' << "\n";
3300                         os << "symbol " << '"'
3301                            << convert_command_inset_arg(p.verbatim_item());
3302                         os << "\"\ndescription \""
3303                            << convert_command_inset_arg(p.verbatim_item())
3304                            << "\"\n";
3305                         end_inset(os);
3306                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3307                 }
3308
3309                 else if (t.cs() == "label") {
3310                         context.check_layout(os);
3311                         begin_command_inset(os, "label", "label");
3312                         os << "name \""
3313                            << convert_command_inset_arg(p.verbatim_item())
3314                            << "\"\n";
3315                         end_inset(os);
3316                 }
3317
3318                 else if (t.cs() == "printindex") {
3319                         context.check_layout(os);
3320                         begin_command_inset(os, "index_print", "printindex");
3321                         os << "type \"idx\"\n";
3322                         end_inset(os);
3323                         skip_spaces_braces(p);
3324                         preamble.registerAutomaticallyLoadedPackage("makeidx");
3325                         if (preamble.use_indices() == "true")
3326                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3327                 }
3328
3329                 else if (t.cs() == "printnomenclature") {
3330                         string width = "";
3331                         string width_type = "";
3332                         context.check_layout(os);
3333                         begin_command_inset(os, "nomencl_print", "printnomenclature");
3334                         // case of a custom width
3335                         if (p.hasOpt()) {
3336                                 width = p.getArg('[', ']');
3337                                 width = translate_len(width);
3338                                 width_type = "custom";
3339                         }
3340                         // case of no custom width
3341                         // the case of no custom width but the width set
3342                         // via \settowidth{\nomlabelwidth}{***} cannot be supported
3343                         // because the user could have set anything, not only the width
3344                         // of the longest label (which would be width_type = "auto")
3345                         string label = convert_command_inset_arg(p.getArg('{', '}'));
3346                         if (label.empty() && width_type.empty())
3347                                 width_type = "none";
3348                         os << "set_width \"" << width_type << "\"\n";
3349                         if (width_type == "custom")
3350                                 os << "width \"" << width << '\"';
3351                         end_inset(os);
3352                         skip_spaces_braces(p);
3353                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3354                 }
3355
3356                 else if ((t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
3357                         context.check_layout(os);
3358                         begin_inset(os, "script ");
3359                         os << t.cs().substr(4) << '\n';
3360                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
3361                         end_inset(os);
3362                         if (t.cs() == "textsubscript")
3363                                 preamble.registerAutomaticallyLoadedPackage("subscript");
3364                 }
3365
3366                 else if ((where = is_known(t.cs(), known_quotes))) {
3367                         context.check_layout(os);
3368                         begin_inset(os, "Quotes ");
3369                         os << known_coded_quotes[where - known_quotes];
3370                         end_inset(os);
3371                         // LyX adds {} after the quote, so we have to eat
3372                         // spaces here if there are any before a possible
3373                         // {} pair.
3374                         eat_whitespace(p, os, context, false);
3375                         skip_braces(p);
3376                 }
3377
3378                 else if ((where = is_known(t.cs(), known_sizes)) &&
3379                          context.new_layout_allowed) {
3380                         context.check_layout(os);
3381                         TeXFont const oldFont = context.font;
3382                         context.font.size = known_coded_sizes[where - known_sizes];
3383                         output_font_change(os, oldFont, context.font);
3384                         eat_whitespace(p, os, context, false);
3385                 }
3386
3387                 else if ((where = is_known(t.cs(), known_font_families)) &&
3388                          context.new_layout_allowed) {
3389                         context.check_layout(os);
3390                         TeXFont const oldFont = context.font;
3391                         context.font.family =
3392                                 known_coded_font_families[where - known_font_families];
3393                         output_font_change(os, oldFont, context.font);
3394                         eat_whitespace(p, os, context, false);
3395                 }
3396
3397                 else if ((where = is_known(t.cs(), known_font_series)) &&
3398                          context.new_layout_allowed) {
3399                         context.check_layout(os);
3400                         TeXFont const oldFont = context.font;
3401                         context.font.series =
3402                                 known_coded_font_series[where - known_font_series];
3403                         output_font_change(os, oldFont, context.font);
3404                         eat_whitespace(p, os, context, false);
3405                 }
3406
3407                 else if ((where = is_known(t.cs(), known_font_shapes)) &&
3408                          context.new_layout_allowed) {
3409                         context.check_layout(os);
3410                         TeXFont const oldFont = context.font;
3411                         context.font.shape =
3412                                 known_coded_font_shapes[where - known_font_shapes];
3413                         output_font_change(os, oldFont, context.font);
3414                         eat_whitespace(p, os, context, false);
3415                 }
3416                 else if ((where = is_known(t.cs(), known_old_font_families)) &&
3417                          context.new_layout_allowed) {
3418                         context.check_layout(os);
3419                         TeXFont const oldFont = context.font;
3420                         context.font.init();
3421                         context.font.size = oldFont.size;
3422                         context.font.family =
3423                                 known_coded_font_families[where - known_old_font_families];
3424                         output_font_change(os, oldFont, context.font);
3425                         eat_whitespace(p, os, context, false);
3426                 }
3427
3428                 else if ((where = is_known(t.cs(), known_old_font_series)) &&
3429                          context.new_layout_allowed) {
3430                         context.check_layout(os);
3431                         TeXFont const oldFont = context.font;
3432                         context.font.init();
3433                         context.font.size = oldFont.size;
3434                         context.font.series =
3435                                 known_coded_font_series[where - known_old_font_series];
3436                         output_font_change(os, oldFont, context.font);
3437                         eat_whitespace(p, os, context, false);
3438                 }
3439
3440                 else if ((where = is_known(t.cs(), known_old_font_shapes)) &&
3441                          context.new_layout_allowed) {
3442                         context.check_layout(os);
3443                         TeXFont const oldFont = context.font;
3444                         context.font.init();
3445                         context.font.size = oldFont.size;
3446                         context.font.shape =
3447                                 known_coded_font_shapes[where - known_old_font_shapes];
3448                         output_font_change(os, oldFont, context.font);
3449                         eat_whitespace(p, os, context, false);
3450                 }
3451
3452                 else if (t.cs() == "selectlanguage") {
3453                         context.check_layout(os);
3454                         // save the language for the case that a
3455                         // \foreignlanguage is used
3456                         context.font.language = babel2lyx(p.verbatim_item());
3457                         os << "\n\\lang " << context.font.language << "\n";
3458                 }
3459
3460                 else if (t.cs() == "foreignlanguage") {
3461                         string const lang = babel2lyx(p.verbatim_item());
3462                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3463                                               context, "\\lang",
3464                                               context.font.language, lang);
3465                 }
3466
3467                 else if (is_known(t.cs().substr(4, string::npos), polyglossia_languages)) {
3468                         // scheme is \textLANGUAGE{text} where LANGUAGE is in polyglossia_languages[]
3469                         string lang;
3470                         // We have to output the whole command if it has an option
3471                         // because LyX doesn't support this yet, see bug #8214,
3472                         // only if there is a single option specifying a variant, we can handle it.
3473                         if (p.hasOpt()) {
3474                                 string langopts = p.getOpt();
3475                                 // check if the option contains a variant, if yes, extract it
3476                                 string::size_type pos_var = langopts.find("variant");
3477                                 string::size_type i = langopts.find(',');
3478                                 if (pos_var != string::npos){
3479                                         string variant;
3480                                         if (i == string::npos) {
3481                                                 variant = langopts.substr(pos_var + 8, langopts.length() - pos_var - 9);
3482                                                 lang = polyglossia2lyx(variant);
3483                                                 parse_text_attributes(p, os, FLAG_ITEM, outer,
3484                                                                           context, "\\lang",
3485                                                                           context.font.language, lang);
3486                                         }
3487                                         else
3488                                                 handle_ert(os, t.asInput() + langopts, context);
3489                                 } else
3490                                         handle_ert(os, t.asInput() + langopts, context);
3491                         } else {
3492                                 lang = polyglossia2lyx(t.cs().substr(4, string::npos));
3493                                 parse_text_attributes(p, os, FLAG_ITEM, outer,
3494                                                           context, "\\lang",
3495                                                           context.font.language, lang);
3496                         }
3497                 }
3498
3499                 else if (t.cs() == "inputencoding") {
3500                         // nothing to write here
3501                         string const enc = subst(p.verbatim_item(), "\n", " ");
3502                         p.setEncoding(enc);
3503                 }
3504
3505                 else if ((where = is_known(t.cs(), known_special_chars))) {
3506                         context.check_layout(os);
3507                         os << "\\SpecialChar \\"
3508                            << known_coded_special_chars[where - known_special_chars]
3509                            << '\n';
3510                         skip_spaces_braces(p);
3511                 }
3512
3513                 else if (t.cs() == "nobreakdash" && p.next_token().asInput() == "-") {
3514                         context.check_layout(os);
3515                         os << "\\SpecialChar \\nobreakdash-\n";
3516                         p.get_token();
3517                 }
3518
3519                 else if (t.cs() == "textquotedbl") {
3520                         context.check_layout(os);
3521                         os << "\"";
3522                         skip_braces(p);
3523                 }
3524
3525                 else if (t.cs() == "@" && p.next_token().asInput() == ".") {
3526                         context.check_layout(os);
3527                         os << "\\SpecialChar \\@.\n";
3528                         p.get_token();
3529                 }
3530
3531                 else if (t.cs() == "-") {
3532                         context.check_layout(os);
3533                         os << "\\SpecialChar \\-\n";
3534                 }
3535
3536                 else if (t.cs() == "textasciitilde") {
3537                         context.check_layout(os);
3538                         os << '~';
3539                         skip_spaces_braces(p);
3540                 }
3541
3542                 else if (t.cs() == "textasciicircum") {
3543                         context.check_layout(os);
3544                         os << '^';
3545                         skip_spaces_braces(p);
3546                 }
3547
3548                 else if (t.cs() == "textbackslash") {
3549                         context.check_layout(os);
3550                         os << "\n\\backslash\n";
3551                         skip_spaces_braces(p);
3552                 }
3553
3554                 else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#"
3555                             || t.cs() == "$" || t.cs() == "{" || t.cs() == "}"
3556                             || t.cs() == "%") {
3557                         context.check_layout(os);
3558                         os << t.cs();
3559                 }
3560
3561                 else if (t.cs() == "char") {
3562                         context.check_layout(os);
3563                         if (p.next_token().character() == '`') {
3564                                 p.get_token();
3565                                 if (p.next_token().cs() == "\"") {
3566                                         p.get_token();
3567                                         os << '"';
3568                                         skip_braces(p);
3569                                 } else {
3570                                         handle_ert(os, "\\char`", context);
3571                                 }
3572                         } else {
3573                                 handle_ert(os, "\\char", context);
3574                         }
3575                 }
3576
3577                 else if (t.cs() == "verb") {
3578                         context.check_layout(os);
3579                         char const delimiter = p.next_token().character();
3580                         string const arg = p.getArg(delimiter, delimiter);
3581                         ostringstream oss;
3582                         oss << "\\verb" << delimiter << arg << delimiter;
3583                         handle_ert(os, oss.str(), context);
3584                 }
3585
3586                 // Problem: \= creates a tabstop inside the tabbing environment
3587                 // and else an accent. In the latter case we really would want
3588                 // \={o} instead of \= o.
3589                 else if (t.cs() == "=" && (flags & FLAG_TABBING))
3590                         handle_ert(os, t.asInput(), context);
3591
3592                 // accents (see Table 6 in Comprehensive LaTeX Symbol List)
3593                 else if (t.cs().size() == 1
3594                          && contains("\"'.=^`bcdHkrtuv~", t.cs())) {
3595                         context.check_layout(os);
3596                         // try to see whether the string is in unicodesymbols
3597                         bool termination;
3598                         docstring rem;
3599                         string command = t.asInput() + "{"
3600                                 + trimSpaceAndEol(p.verbatim_item())
3601                                 + "}";
3602                         set<string> req;
3603                         docstring s = encodings.fromLaTeXCommand(from_utf8(command),
3604                                 Encodings::TEXT_CMD | Encodings::MATH_CMD,
3605                                 termination, rem, &req);
3606                         if (!s.empty()) {
3607                                 if (!rem.empty())
3608                                         cerr << "When parsing " << command
3609                                              << ", result is " << to_utf8(s)
3610                                              << "+" << to_utf8(rem) << endl;
3611                                 os << to_utf8(s);
3612                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
3613                                         preamble.registerAutomaticallyLoadedPackage(*it);
3614                         } else
3615                                 // we did not find a non-ert version
3616                                 handle_ert(os, command, context);
3617                 }
3618
3619                 else if (t.cs() == "\\") {
3620                         context.check_layout(os);
3621                         if (p.hasOpt())
3622                                 handle_ert(os, "\\\\" + p.getOpt(), context);
3623                         else if (p.next_token().asInput() == "*") {
3624                                 p.get_token();
3625                                 // getOpt() eats the following space if there
3626                                 // is no optional argument, but that is OK
3627                                 // here since it has no effect in the output.
3628                                 handle_ert(os, "\\\\*" + p.getOpt(), context);
3629                         }
3630                         else {
3631                                 begin_inset(os, "Newline newline");
3632                                 end_inset(os);
3633                         }
3634                 }
3635
3636                 else if (t.cs() == "newline" ||
3637                          (t.cs() == "linebreak" && !p.hasOpt())) {
3638                         context.check_layout(os);
3639                         begin_inset(os, "Newline ");
3640                         os << t.cs();
3641                         end_inset(os);
3642                         skip_spaces_braces(p);
3643                 }
3644
3645                 else if (t.cs() == "input" || t.cs() == "include"
3646                          || t.cs() == "verbatiminput") {
3647                         string name = t.cs();
3648                         if (t.cs() == "verbatiminput"
3649                             && p.next_token().asInput() == "*")
3650                                 name += p.get_token().asInput();
3651                         context.check_layout(os);
3652                         string filename(normalize_filename(p.getArg('{', '}')));
3653                         string const path = getMasterFilePath();
3654                         // We want to preserve relative / absolute filenames,
3655                         // therefore path is only used for testing
3656                         if ((t.cs() == "include" || t.cs() == "input") &&
3657                             !makeAbsPath(filename, path).exists()) {
3658                                 // The file extension is probably missing.
3659                                 // Now try to find it out.
3660                                 string const tex_name =
3661                                         find_file(filename, path,
3662                                                   known_tex_extensions);
3663                                 if (!tex_name.empty())
3664                                         filename = tex_name;
3665                         }
3666                         bool external = false;
3667                         string outname;
3668                         if (makeAbsPath(filename, path).exists()) {
3669                                 string const abstexname =
3670                                         makeAbsPath(filename, path).absFileName();
3671                                 string const abslyxname =
3672                                         changeExtension(abstexname, ".lyx");
3673                                 string const absfigname =
3674                                         changeExtension(abstexname, ".fig");
3675                                 fix_relative_filename(filename);
3676                                 string const lyxname =
3677                                         changeExtension(filename, ".lyx");
3678                                 bool xfig = false;
3679                                 external = FileName(absfigname).exists();
3680                                 if (t.cs() == "input") {
3681                                         string const ext = getExtension(abstexname);
3682
3683                                         // Combined PS/LaTeX:
3684                                         // x.eps, x.pstex_t (old xfig)
3685                                         // x.pstex, x.pstex_t (new xfig, e.g. 3.2.5)
3686                                         FileName const absepsname(
3687                                                 changeExtension(abstexname, ".eps"));
3688                                         FileName const abspstexname(
3689                                                 changeExtension(abstexname, ".pstex"));
3690                                         bool const xfigeps =
3691                                                 (absepsname.exists() ||
3692                                                  abspstexname.exists()) &&
3693                                                 ext == "pstex_t";
3694
3695                                         // Combined PDF/LaTeX:
3696                                         // x.pdf, x.pdftex_t (old xfig)
3697                                         // x.pdf, x.pdf_t (new xfig, e.g. 3.2.5)
3698                                         FileName const abspdfname(
3699                                                 changeExtension(abstexname, ".pdf"));
3700                                         bool const xfigpdf =
3701                                                 abspdfname.exists() &&
3702                                                 (ext == "pdftex_t" || ext == "pdf_t");
3703                                         if (xfigpdf)
3704                                                 pdflatex = true;
3705
3706                                         // Combined PS/PDF/LaTeX:
3707                                         // x_pspdftex.eps, x_pspdftex.pdf, x.pspdftex
3708                                         string const absbase2(
3709                                                 removeExtension(abstexname) + "_pspdftex");
3710                                         FileName const abseps2name(
3711                                                 addExtension(absbase2, ".eps"));
3712                                         FileName const abspdf2name(
3713                                                 addExtension(absbase2, ".pdf"));
3714                                         bool const xfigboth =
3715                                                 abspdf2name.exists() &&
3716                                                 abseps2name.exists() && ext == "pspdftex";
3717
3718                                         xfig = xfigpdf || xfigeps || xfigboth;
3719                                         external = external && xfig;
3720                                 }
3721                                 if (external) {
3722                                         outname = changeExtension(filename, ".fig");
3723                                 } else if (xfig) {
3724                                         // Don't try to convert, the result
3725                                         // would be full of ERT.
3726                                         outname = filename;
3727                                 } else if (t.cs() != "verbatiminput" &&
3728                                     tex2lyx(abstexname, FileName(abslyxname),
3729                                             p.getEncoding())) {
3730                                         outname = lyxname;
3731                                 } else {
3732                                         outname = filename;
3733                                 }
3734                         } else {
3735                                 cerr << "Warning: Could not find included file '"
3736                                      << filename << "'." << endl;
3737                                 outname = filename;
3738                         }
3739                         if (external) {
3740                                 begin_inset(os, "External\n");
3741                                 os << "\ttemplate XFig\n"
3742                                    << "\tfilename " << outname << '\n';
3743                                 registerExternalTemplatePackages("XFig");
3744                         } else {
3745                                 begin_command_inset(os, "include", name);
3746                                 os << "preview false\n"
3747                                       "filename \"" << outname << "\"\n";
3748                                 if (t.cs() == "verbatiminput")
3749                                         preamble.registerAutomaticallyLoadedPackage("verbatim");
3750                         }
3751                         end_inset(os);
3752                 }
3753
3754                 else if (t.cs() == "bibliographystyle") {
3755                         // store new bibliographystyle
3756                         bibliographystyle = p.verbatim_item();
3757                         // If any other command than \bibliography and
3758                         // \nocite{*} follows, we need to output the style
3759                         // (because it might be used by that command).
3760                         // Otherwise, it will automatically be output by LyX.
3761                         p.pushPosition();
3762                         bool output = true;
3763                         for (Token t2 = p.get_token(); p.good(); t2 = p.get_token()) {
3764                                 if (t2.cat() == catBegin)
3765                                         break;
3766                                 if (t2.cat() != catEscape)
3767                                         continue;
3768                                 if (t2.cs() == "nocite") {
3769                                         if (p.getArg('{', '}') == "*")
3770                                                 continue;
3771                                 } else if (t2.cs() == "bibliography")
3772                                         output = false;
3773                                 break;
3774                         }
3775                         p.popPosition();
3776                         if (output) {
3777                                 handle_ert(os,
3778                                         "\\bibliographystyle{" + bibliographystyle + '}',
3779                                         context);
3780                         }
3781                 }
3782
3783                 else if (t.cs() == "bibliography") {
3784                         context.check_layout(os);
3785                         begin_command_inset(os, "bibtex", "bibtex");
3786                         if (!btprint.empty()) {
3787                                 os << "btprint " << '"' << "btPrintAll" << '"' << "\n";
3788                                 // clear the string because the next BibTeX inset can be without the
3789                                 // \nocite{*} option
3790                                 btprint.clear();
3791                         }
3792                         os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n";
3793                         // Do we have a bibliographystyle set?
3794                         if (!bibliographystyle.empty())
3795                                 os << "options " << '"' << bibliographystyle << '"' << "\n";
3796                         end_inset(os);
3797                 }
3798
3799                 else if (t.cs() == "parbox") {
3800                         // Test whether this is an outer box of a shaded box
3801                         p.pushPosition();
3802                         // swallow arguments
3803                         while (p.hasOpt()) {
3804                                 p.getArg('[', ']');
3805                                 p.skip_spaces(true);
3806                         }
3807                         p.getArg('{', '}');
3808                         p.skip_spaces(true);
3809                         // eat the '{'
3810                         if (p.next_token().cat() == catBegin)
3811                                 p.get_token();
3812                         p.skip_spaces(true);
3813                         Token to = p.get_token();
3814                         bool shaded = false;
3815                         if (to.asInput() == "\\begin") {
3816                                 p.skip_spaces(true);
3817                                 if (p.getArg('{', '}') == "shaded")
3818                                         shaded = true;
3819                         }
3820                         p.popPosition();
3821                         if (shaded) {
3822                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3823                                                 context, "parbox", "shaded");
3824                         } else
3825                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3826                                           "", "", t.cs());
3827                 }
3828
3829                 else if (t.cs() == "ovalbox" || t.cs() == "Ovalbox" ||
3830                          t.cs() == "shadowbox" || t.cs() == "doublebox")
3831                         parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), "");
3832
3833                 else if (t.cs() == "framebox") {
3834                         if (p.next_token().character() == '(') {
3835                                 //the syntax is: \framebox(x,y)[position]{content}
3836                                 string arg = t.asInput();
3837                                 arg += p.getFullParentheseArg();
3838                                 arg += p.getFullOpt();
3839                                 eat_whitespace(p, os, context, false);
3840                                 handle_ert(os, arg + '{', context);
3841                                 eat_whitespace(p, os, context, false);
3842                                 parse_text(p, os, FLAG_ITEM, outer, context);
3843                                 handle_ert(os, "}", context);
3844                         } else {
3845                                 string special = p.getFullOpt();
3846                                 special += p.getOpt();
3847                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3848                                                 context, t.cs(), special);
3849                         }
3850                 }
3851
3852                 //\makebox() is part of the picture environment and different from \makebox{}
3853                 //\makebox{} will be parsed by parse_box
3854                 else if (t.cs() == "makebox") {
3855                         if (p.next_token().character() == '(') {
3856                                 //the syntax is: \makebox(x,y)[position]{content}
3857                                 string arg = t.asInput();
3858                                 arg += p.getFullParentheseArg();
3859                                 arg += p.getFullOpt();
3860                                 eat_whitespace(p, os, context, false);
3861                                 handle_ert(os, arg + '{', context);
3862                                 eat_whitespace(p, os, context, false);
3863                                 parse_text(p, os, FLAG_ITEM, outer, context);
3864                                 handle_ert(os, "}", context);
3865                         } else
3866                                 //the syntax is: \makebox[width][position]{content}
3867                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3868                                           "", "", t.cs());
3869                 }
3870
3871                 else if (t.cs() == "smallskip" ||
3872                          t.cs() == "medskip" ||
3873                          t.cs() == "bigskip" ||
3874                          t.cs() == "vfill") {
3875                         context.check_layout(os);
3876                         begin_inset(os, "VSpace ");
3877                         os << t.cs();
3878                         end_inset(os);
3879                         skip_spaces_braces(p);
3880                 }
3881
3882                 else if ((where = is_known(t.cs(), known_spaces))) {
3883                         context.check_layout(os);
3884                         begin_inset(os, "space ");
3885                         os << '\\' << known_coded_spaces[where - known_spaces]
3886                            << '\n';
3887                         end_inset(os);
3888                         // LaTeX swallows whitespace after all spaces except
3889                         // "\\,". We have to do that here, too, because LyX
3890                         // adds "{}" which would make the spaces significant.
3891                         if (t.cs() !=  ",")
3892                                 eat_whitespace(p, os, context, false);
3893                         // LyX adds "{}" after all spaces except "\\ " and
3894                         // "\\,", so we have to remove "{}".
3895                         // "\\,{}" is equivalent to "\\," in LaTeX, so we
3896                         // remove the braces after "\\,", too.
3897                         if (t.cs() != " ")
3898                                 skip_braces(p);
3899                 }
3900
3901                 else if (t.cs() == "newpage" ||
3902                          (t.cs() == "pagebreak" && !p.hasOpt()) ||
3903                          t.cs() == "clearpage" ||
3904                          t.cs() == "cleardoublepage") {
3905                         context.check_layout(os);
3906                         begin_inset(os, "Newpage ");
3907                         os << t.cs();
3908                         end_inset(os);
3909                         skip_spaces_braces(p);
3910                 }
3911
3912                 else if (t.cs() == "DeclareRobustCommand" ||
3913                          t.cs() == "DeclareRobustCommandx" ||
3914                          t.cs() == "newcommand" ||
3915                          t.cs() == "newcommandx" ||
3916                          t.cs() == "providecommand" ||
3917                          t.cs() == "providecommandx" ||
3918                          t.cs() == "renewcommand" ||
3919                          t.cs() == "renewcommandx") {
3920                         // DeclareRobustCommand, DeclareRobustCommandx,
3921                         // providecommand and providecommandx could be handled
3922                         // by parse_command(), but we need to call
3923                         // add_known_command() here.
3924                         string name = t.asInput();
3925                         if (p.next_token().asInput() == "*") {
3926                                 // Starred form. Eat '*'
3927                                 p.get_token();
3928                                 name += '*';
3929                         }
3930                         string const command = p.verbatim_item();
3931                         string const opt1 = p.getFullOpt();
3932                         string const opt2 = p.getFullOpt();
3933                         add_known_command(command, opt1, !opt2.empty());
3934                         string const ert = name + '{' + command + '}' +
3935                                            opt1 + opt2 +
3936                                            '{' + p.verbatim_item() + '}';
3937
3938                         if (t.cs() == "DeclareRobustCommand" ||
3939                             t.cs() == "DeclareRobustCommandx" ||
3940                             t.cs() == "providecommand" ||
3941                             t.cs() == "providecommandx" ||
3942                             name[name.length()-1] == '*')
3943                                 handle_ert(os, ert, context);
3944                         else {
3945                                 context.check_layout(os);
3946                                 begin_inset(os, "FormulaMacro");
3947                                 os << "\n" << ert;
3948                                 end_inset(os);
3949                         }
3950                 }
3951
3952                 else if (t.cs() == "let" && p.next_token().asInput() != "*") {
3953                         // let could be handled by parse_command(),
3954                         // but we need to call add_known_command() here.
3955                         string ert = t.asInput();
3956                         string name;
3957                         p.skip_spaces();
3958                         if (p.next_token().cat() == catBegin) {
3959                                 name = p.verbatim_item();
3960                                 ert += '{' + name + '}';
3961                         } else {
3962                                 name = p.verbatim_item();
3963                                 ert += name;
3964                         }
3965                         string command;
3966                         p.skip_spaces();
3967                         if (p.next_token().cat() == catBegin) {
3968                                 command = p.verbatim_item();
3969                                 ert += '{' + command + '}';
3970                         } else {
3971                                 command = p.verbatim_item();
3972                                 ert += command;
3973                         }
3974                         // If command is known, make name known too, to parse
3975                         // its arguments correctly. For this reason we also
3976                         // have commands in syntax.default that are hardcoded.
3977                         CommandMap::iterator it = known_commands.find(command);
3978                         if (it != known_commands.end())
3979                                 known_commands[t.asInput()] = it->second;
3980                         handle_ert(os, ert, context);
3981                 }
3982
3983                 else if (t.cs() == "hspace" || t.cs() == "vspace") {
3984                         bool starred = false;
3985                         if (p.next_token().asInput() == "*") {
3986                                 p.get_token();
3987                                 starred = true;
3988                         }
3989                         string name = t.asInput();
3990                         string const length = p.verbatim_item();
3991                         string unit;
3992                         string valstring;
3993                         bool valid = splitLatexLength(length, valstring, unit);
3994                         bool known_hspace = false;
3995                         bool known_vspace = false;
3996                         bool known_unit = false;
3997                         double value;
3998                         if (valid) {
3999                                 istringstream iss(valstring);
4000                                 iss >> value;
4001                                 if (value == 1.0) {
4002                                         if (t.cs()[0] == 'h') {
4003                                                 if (unit == "\\fill") {
4004                                                         if (!starred) {
4005                                                                 unit = "";
4006                                                                 name = "\\hfill";
4007                                                         }
4008                                                         known_hspace = true;
4009                                                 }
4010                                         } else {
4011                                                 if (unit == "\\smallskipamount") {
4012                                                         unit = "smallskip";
4013                                                         known_vspace = true;
4014                                                 } else if (unit == "\\medskipamount") {
4015                                                         unit = "medskip";
4016                                                         known_vspace = true;
4017                                                 } else if (unit == "\\bigskipamount") {
4018                                                         unit = "bigskip";
4019                                                         known_vspace = true;
4020                                                 } else if (unit == "\\fill") {
4021                                                         unit = "vfill";
4022                                                         known_vspace = true;
4023                                                 }
4024                                         }
4025                                 }
4026                                 if (!known_hspace && !known_vspace) {
4027                                         switch (unitFromString(unit)) {
4028                                         case Length::SP:
4029                                         case Length::PT:
4030                                         case Length::BP:
4031                                         case Length::DD:
4032                                         case Length::MM:
4033                                         case Length::PC:
4034                                         case Length::CC:
4035                                         case Length::CM:
4036                                         case Length::IN:
4037                                         case Length::EX:
4038                                         case Length::EM:
4039                                         case Length::MU:
4040                                                 known_unit = true;
4041                                                 break;
4042                                         default:
4043                                                 break;
4044                                         }
4045                                 }
4046                         }
4047
4048                         if (t.cs()[0] == 'h' && (known_unit || known_hspace)) {
4049                                 // Literal horizontal length or known variable
4050                                 context.check_layout(os);
4051                                 begin_inset(os, "space ");
4052                                 os << name;
4053                                 if (starred)
4054                                         os << '*';
4055                                 os << '{';
4056                                 if (known_hspace)
4057                                         os << unit;
4058                                 os << "}";
4059                                 if (known_unit && !known_hspace)
4060                                         os << "\n\\length "
4061                                            << translate_len(length);
4062                                 end_inset(os);
4063                         } else if (known_unit || known_vspace) {
4064                                 // Literal vertical length or known variable
4065                                 context.check_layout(os);
4066                                 begin_inset(os, "VSpace ");
4067                                 if (known_unit)
4068                                         os << value;
4069                                 os << unit;
4070                                 if (starred)
4071                                         os << '*';
4072                                 end_inset(os);
4073                         } else {
4074                                 // LyX can't handle other length variables in Inset VSpace/space
4075                                 if (starred)
4076                                         name += '*';
4077                                 if (valid) {
4078                                         if (value == 1.0)
4079                                                 handle_ert(os, name + '{' + unit + '}', context);
4080                                         else if (value == -1.0)
4081                                                 handle_ert(os, name + "{-" + unit + '}', context);
4082                                         else
4083                                                 handle_ert(os, name + '{' + valstring + unit + '}', context);
4084                                 } else
4085                                         handle_ert(os, name + '{' + length + '}', context);
4086                         }
4087                 }
4088
4089                 // The single '=' is meant here.
4090                 else if ((newinsetlayout = findInsetLayout(context.textclass, t.cs(), true))) {
4091                         p.skip_spaces();
4092                         context.check_layout(os);
4093                         begin_inset(os, "Flex ");
4094                         os << to_utf8(newinsetlayout->name()) << '\n'
4095                            << "status collapsed\n";
4096                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
4097                         end_inset(os);
4098                 }
4099
4100                 else if (t.cs() == "includepdf") {
4101                         p.skip_spaces();
4102                         string const arg = p.getArg('[', ']');
4103                         map<string, string> opts;
4104                         vector<string> keys;
4105                         split_map(arg, opts, keys);
4106                         string name = normalize_filename(p.verbatim_item());
4107                         string const path = getMasterFilePath();
4108                         // We want to preserve relative / absolute filenames,
4109                         // therefore path is only used for testing
4110                         if (!makeAbsPath(name, path).exists()) {
4111                                 // The file extension is probably missing.
4112                                 // Now try to find it out.
4113                                 char const * const pdfpages_format[] = {"pdf", 0};
4114                                 string const pdftex_name =
4115                                         find_file(name, path, pdfpages_format);
4116                                 if (!pdftex_name.empty()) {
4117                                         name = pdftex_name;
4118                                         pdflatex = true;
4119                                 }
4120                         }
4121                         if (makeAbsPath(name, path).exists())
4122                                 fix_relative_filename(name);
4123                         else
4124                                 cerr << "Warning: Could not find file '"
4125                                      << name << "'." << endl;
4126                         // write output
4127                         context.check_layout(os);
4128                         begin_inset(os, "External\n\ttemplate ");
4129                         os << "PDFPages\n\tfilename "
4130                            << name << "\n";
4131                         // parse the options
4132                         if (opts.find("pages") != opts.end())
4133                                 os << "\textra LaTeX \"pages="
4134                                    << opts["pages"] << "\"\n";
4135                         if (opts.find("angle") != opts.end())
4136                                 os << "\trotateAngle "
4137                                    << opts["angle"] << '\n';
4138                         if (opts.find("origin") != opts.end()) {
4139                                 ostringstream ss;
4140                                 string const opt = opts["origin"];
4141                                 if (opt == "tl") ss << "topleft";
4142                                 if (opt == "bl") ss << "bottomleft";
4143                                 if (opt == "Bl") ss << "baselineleft";
4144                                 if (opt == "c") ss << "center";
4145                                 if (opt == "tc") ss << "topcenter";
4146                                 if (opt == "bc") ss << "bottomcenter";
4147                                 if (opt == "Bc") ss << "baselinecenter";
4148                                 if (opt == "tr") ss << "topright";
4149                                 if (opt == "br") ss << "bottomright";
4150                                 if (opt == "Br") ss << "baselineright";
4151                                 if (!ss.str().empty())
4152                                         os << "\trotateOrigin " << ss.str() << '\n';
4153                                 else
4154                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
4155                         }
4156                         if (opts.find("width") != opts.end())
4157                                 os << "\twidth "
4158                                    << translate_len(opts["width"]) << '\n';
4159                         if (opts.find("height") != opts.end())
4160                                 os << "\theight "
4161                                    << translate_len(opts["height"]) << '\n';
4162                         if (opts.find("keepaspectratio") != opts.end())
4163                                 os << "\tkeepAspectRatio\n";
4164                         end_inset(os);
4165                         context.check_layout(os);
4166                         registerExternalTemplatePackages("PDFPages");
4167                 }
4168
4169                 else if (t.cs() == "loadgame") {
4170                         p.skip_spaces();
4171                         string name = normalize_filename(p.verbatim_item());
4172                         string const path = getMasterFilePath();
4173                         // We want to preserve relative / absolute filenames,
4174                         // therefore path is only used for testing
4175                         if (!makeAbsPath(name, path).exists()) {
4176                                 // The file extension is probably missing.
4177                                 // Now try to find it out.
4178                                 char const * const lyxskak_format[] = {"fen", 0};
4179                                 string const lyxskak_name =
4180                                         find_file(name, path, lyxskak_format);
4181                                 if (!lyxskak_name.empty())
4182                                         name = lyxskak_name;
4183                         }
4184                         if (makeAbsPath(name, path).exists())
4185                                 fix_relative_filename(name);
4186                         else
4187                                 cerr << "Warning: Could not find file '"
4188                                      << name << "'." << endl;
4189                         context.check_layout(os);
4190                         begin_inset(os, "External\n\ttemplate ");
4191                         os << "ChessDiagram\n\tfilename "
4192                            << name << "\n";
4193                         end_inset(os);
4194                         context.check_layout(os);
4195                         // after a \loadgame follows a \showboard
4196                         if (p.get_token().asInput() == "showboard")
4197                                 p.get_token();
4198                         registerExternalTemplatePackages("ChessDiagram");
4199                 }
4200
4201                 else {
4202                         // try to see whether the string is in unicodesymbols
4203                         // Only use text mode commands, since we are in text mode here,
4204                         // and math commands may be invalid (bug 6797)
4205                         bool termination;
4206                         docstring rem;
4207                         set<string> req;
4208                         docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
4209                                         Encodings::TEXT_CMD, termination, rem, &req);
4210                         if (!s.empty()) {
4211                                 if (!rem.empty())
4212                                         cerr << "When parsing " << t.cs()
4213                                              << ", result is " << to_utf8(s)
4214                                              << "+" << to_utf8(rem) << endl;
4215                                 context.check_layout(os);
4216                                 os << to_utf8(s);
4217                                 if (termination)
4218                                         skip_spaces_braces(p);
4219                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
4220                                         preamble.registerAutomaticallyLoadedPackage(*it);
4221                         }
4222                         //cerr << "#: " << t << " mode: " << mode << endl;
4223                         // heuristic: read up to next non-nested space
4224                         /*
4225                         string s = t.asInput();
4226                         string z = p.verbatim_item();
4227                         while (p.good() && z != " " && z.size()) {
4228                                 //cerr << "read: " << z << endl;
4229                                 s += z;
4230                                 z = p.verbatim_item();
4231                         }
4232                         cerr << "found ERT: " << s << endl;
4233                         handle_ert(os, s + ' ', context);
4234                         */
4235                         else {
4236                                 string name = t.asInput();
4237                                 if (p.next_token().asInput() == "*") {
4238                                         // Starred commands like \vspace*{}
4239                                         p.get_token();  // Eat '*'
4240                                         name += '*';
4241                                 }
4242                                 if (!parse_command(name, p, os, outer, context))
4243                                         handle_ert(os, name, context);
4244                         }
4245                 }
4246
4247                 if (flags & FLAG_LEAVE) {
4248                         flags &= ~FLAG_LEAVE;
4249                         break;
4250                 }
4251         }
4252 }
4253
4254 // }])
4255
4256
4257 } // namespace lyx