src/tex2lyx/text.cpp

   1 /**
   2  * \file tex2lyx/text.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Jean-Marc Lasgouttes
   8  * \author Uwe Stöhr
   9  *
  10  * Full author contact details are available in file CREDITS.
  11  */
  12
  13 // {[(
  14
  15 #include <config.h>
  16
  17 #include "tex2lyx.h"
  18
  19 #include "Context.h"
  20 #include "Encoding.h"
  21 #include "FloatList.h"
  22 #include "LaTeXPackages.h"
  23 #include "Layout.h"
  24 #include "Length.h"
  25 #include "Preamble.h"
  26
  27 #include "insets/ExternalTemplate.h"
  28
  29 #include "support/lassert.h"
  30 #include "support/convert.h"
  31 #include "support/FileName.h"
  32 #include "support/filetools.h"
  33 #include "support/lstrings.h"
  34 #include "support/lyxtime.h"
  35
  36 #include <algorithm>
  37 #include <iostream>
  38 #include <map>
  39 #include <sstream>
  40 #include <vector>
  41
  42 using namespace std;
  43 using namespace lyx::support;
  44
  45 namespace lyx {
  46
  47
  48 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  49                 Context const & context, InsetLayout const * layout)
  50 {
  51         bool const forcePlainLayout =
  52                 layout ? layout->forcePlainLayout() : false;
  53         Context newcontext(true, context.textclass);
  54         if (forcePlainLayout)
  55                 newcontext.layout = &context.textclass.plainLayout();
  56         else
  57                 newcontext.font = context.font;
  58         parse_text(p, os, flags, outer, newcontext);
  59         newcontext.check_end_layout(os);
  60 }
  61
  62
  63 namespace {
  64
  65 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  66                 Context const & context, string const & name)
  67 {
  68         InsetLayout const * layout = 0;
  69         DocumentClass::InsetLayouts::const_iterator it =
  70                 context.textclass.insetLayouts().find(from_ascii(name));
  71         if (it != context.textclass.insetLayouts().end())
  72                 layout = &(it->second);
  73         parse_text_in_inset(p, os, flags, outer, context, layout);
  74 }
  75
  76 /// parses a paragraph snippet, useful for example for \\emph{...}
  77 void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
  78                 Context & context)
  79 {
  80         Context newcontext(context);
  81         // Don't inherit the paragraph-level extra stuff
  82         newcontext.par_extra_stuff.clear();
  83         parse_text(p, os, flags, outer, newcontext);
  84         // Make sure that we don't create invalid .lyx files
  85         context.need_layout = newcontext.need_layout;
  86         context.need_end_layout = newcontext.need_end_layout;
  87 }
  88
  89
  90 /*!
  91  * Thin wrapper around parse_text_snippet() using a string.
  92  *
  93  * We completely ignore \c context.need_layout and \c context.need_end_layout,
  94  * because our return value is not used directly (otherwise the stream version
  95  * of parse_text_snippet() could be used). That means that the caller needs
  96  * to do layout management manually.
  97  * This is intended to parse text that does not create any layout changes.
  98  */
  99 string parse_text_snippet(Parser & p, unsigned flags, const bool outer,
 100                   Context & context)
 101 {
 102         Context newcontext(context);
 103         newcontext.need_layout = false;
 104         newcontext.need_end_layout = false;
 105         newcontext.new_layout_allowed = false;
 106         // Avoid warning by Context::~Context()
 107         newcontext.par_extra_stuff.clear();
 108         ostringstream os;
 109         parse_text_snippet(p, os, flags, outer, newcontext);
 110         return os.str();
 111 }
 112
 113
 114 char const * const known_ref_commands[] = { "ref", "pageref", "vref",
 115  "vpageref", "prettyref", "eqref", 0 };
 116
 117 char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref",
 118  "vpageref", "formatted", "eqref", 0 };
 119
 120 /**
 121  * known polyglossia language names (including variants)
 122  */
 123 const char * const polyglossia_languages[] = {
 124 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 125 "nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
 126 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 127 "irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
 128 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 129 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
 130 "brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
 131 "lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 132 "welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
 133 "marathi", "spanish",
 134 "american", "ancient", "australian", "british", "monotonic", "newzealand",
 135 "polytonic", 0};
 136
 137 /**
 138  * the same as polyglossia_languages with .lyx names
 139  * please keep this in sync with polyglossia_languages line by line!
 140  */
 141 const char * const coded_polyglossia_languages[] = {
 142 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 143 "nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
 144 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 145 "irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
 146 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 147 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
 148 "brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
 149 "lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 150 "welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
 151 "marathi", "spanish",
 152 "american", "ancientgreek", "australian", "british", "greek", "newzealand",
 153 "polutonikogreek", 0};
 154
 155 /**
 156  * supported CJK encodings
 157  */
 158 const char * const supported_CJK_encodings[] = {
 159 "EUC-JP", "KS", "GB", "UTF8", 0};
 160
 161 /**
 162  * the same as supported_CJK_encodings with their corresponding LyX language name
 163  * please keep this in sync with supported_CJK_encodings line by line!
 164  */
 165 const char * const coded_supported_CJK_encodings[] = {
 166 "japanese-cjk", "korean", "chinese-simplified", "chinese-traditional", 0};
 167
 168 string CJK2lyx(string const & encoding)
 169 {
 170         char const * const * where = is_known(encoding, supported_CJK_encodings);
 171         if (where)
 172                 return coded_supported_CJK_encodings[where - supported_CJK_encodings];
 173         return encoding;
 174 }
 175
 176 /*!
 177  * natbib commands.
 178  * The starred forms are also known except for "citefullauthor",
 179  * "citeyear" and "citeyearpar".
 180  */
 181 char const * const known_natbib_commands[] = { "cite", "citet", "citep",
 182 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 183 "citefullauthor", "Citet", "Citep", "Citealt", "Citealp", "Citeauthor", 0 };
 184
 185 /*!
 186  * jurabib commands.
 187  * No starred form other than "cite*" known.
 188  */
 189 char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
 190 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 191 // jurabib commands not (yet) supported by LyX:
 192 // "fullcite",
 193 // "footcite", "footcitet", "footcitep", "footcitealt", "footcitealp",
 194 // "footciteauthor", "footciteyear", "footciteyearpar",
 195 "citefield", "citetitle", 0 };
 196
 197 /// LaTeX names for quotes
 198 char const * const known_quotes[] = { "dq", "guillemotleft", "flqq", "og",
 199 "guillemotright", "frqq", "fg", "glq", "glqq", "textquoteleft", "grq", "grqq",
 200 "quotedblbase", "textquotedblleft", "quotesinglbase", "textquoteright", "flq",
 201 "guilsinglleft", "frq", "guilsinglright", 0};
 202
 203 /// the same as known_quotes with .lyx names
 204 char const * const known_coded_quotes[] = { "prd", "ard", "ard", "ard",
 205 "ald", "ald", "ald", "gls", "gld", "els", "els", "grd",
 206 "gld", "grd", "gls", "ers", "fls",
 207 "fls", "frs", "frs", 0};
 208
 209 /// LaTeX names for font sizes
 210 char const * const known_sizes[] = { "tiny", "scriptsize", "footnotesize",
 211 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
 212
 213 /// the same as known_sizes with .lyx names
 214 char const * const known_coded_sizes[] = { "tiny", "scriptsize", "footnotesize",
 215 "small", "normal", "large", "larger", "largest", "huge", "giant", 0};
 216
 217 /// LaTeX 2.09 names for font families
 218 char const * const known_old_font_families[] = { "rm", "sf", "tt", 0};
 219
 220 /// LaTeX names for font families
 221 char const * const known_font_families[] = { "rmfamily", "sffamily",
 222 "ttfamily", 0};
 223
 224 /// LaTeX names for font family changing commands
 225 char const * const known_text_font_families[] = { "textrm", "textsf",
 226 "texttt", 0};
 227
 228 /// The same as known_old_font_families, known_font_families and
 229 /// known_text_font_families with .lyx names
 230 char const * const known_coded_font_families[] = { "roman", "sans",
 231 "typewriter", 0};
 232
 233 /// LaTeX 2.09 names for font series
 234 char const * const known_old_font_series[] = { "bf", 0};
 235
 236 /// LaTeX names for font series
 237 char const * const known_font_series[] = { "bfseries", "mdseries", 0};
 238
 239 /// LaTeX names for font series changing commands
 240 char const * const known_text_font_series[] = { "textbf", "textmd", 0};
 241
 242 /// The same as known_old_font_series, known_font_series and
 243 /// known_text_font_series with .lyx names
 244 char const * const known_coded_font_series[] = { "bold", "medium", 0};
 245
 246 /// LaTeX 2.09 names for font shapes
 247 char const * const known_old_font_shapes[] = { "it", "sl", "sc", 0};
 248
 249 /// LaTeX names for font shapes
 250 char const * const known_font_shapes[] = { "itshape", "slshape", "scshape",
 251 "upshape", 0};
 252
 253 /// LaTeX names for font shape changing commands
 254 char const * const known_text_font_shapes[] = { "textit", "textsl", "textsc",
 255 "textup", 0};
 256
 257 /// The same as known_old_font_shapes, known_font_shapes and
 258 /// known_text_font_shapes with .lyx names
 259 char const * const known_coded_font_shapes[] = { "italic", "slanted",
 260 "smallcaps", "up", 0};
 261
 262 /// Known special characters which need skip_spaces_braces() afterwards
 263 char const * const known_special_chars[] = {"ldots", "lyxarrow",
 264 "textcompwordmark", "slash", 0};
 265
 266 /// the same as known_special_chars with .lyx names
 267 char const * const known_coded_special_chars[] = {"ldots{}", "menuseparator",
 268 "textcompwordmark{}", "slash{}", 0};
 269
 270 /*!
 271  * Graphics file extensions known by the dvips driver of the graphics package.
 272  * These extensions are used to complete the filename of an included
 273  * graphics file if it does not contain an extension.
 274  * The order must be the same that latex uses to find a file, because we
 275  * will use the first extension that matches.
 276  * This is only an approximation for the common cases. If we would want to
 277  * do it right in all cases, we would need to know which graphics driver is
 278  * used and know the extensions of every driver of the graphics package.
 279  */
 280 char const * const known_dvips_graphics_formats[] = {"eps", "ps", "eps.gz",
 281 "ps.gz", "eps.Z", "ps.Z", 0};
 282
 283 /*!
 284  * Graphics file extensions known by the pdftex driver of the graphics package.
 285  * \sa known_dvips_graphics_formats
 286  */
 287 char const * const known_pdftex_graphics_formats[] = {"png", "pdf", "jpg",
 288 "mps", "tif", 0};
 289
 290 /*!
 291  * Known file extensions for TeX files as used by \\include.
 292  */
 293 char const * const known_tex_extensions[] = {"tex", 0};
 294
 295 /// spaces known by InsetSpace
 296 char const * const known_spaces[] = { " ", "space", ",",
 297 "thinspace", "quad", "qquad", "enspace", "enskip",
 298 "negthinspace", "negmedspace", "negthickspace", "textvisiblespace",
 299 "hfill", "dotfill", "hrulefill", "leftarrowfill", "rightarrowfill",
 300 "upbracefill", "downbracefill", 0};
 301
 302 /// the same as known_spaces with .lyx names
 303 char const * const known_coded_spaces[] = { "space{}", "space{}",
 304 "thinspace{}", "thinspace{}", "quad{}", "qquad{}", "enspace{}", "enskip{}",
 305 "negthinspace{}", "negmedspace{}", "negthickspace{}", "textvisiblespace{}",
 306 "hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}", "rightarrowfill{}",
 307 "upbracefill{}", "downbracefill{}", 0};
 308
 309 /// These are translated by LyX to commands like "\\LyX{}", so we have to put
 310 /// them in ERT. "LaTeXe" must come before "LaTeX"!
 311 char const * const known_phrases[] = {"LyX", "TeX", "LaTeXe", "LaTeX", 0};
 312 char const * const known_coded_phrases[] = {"LyX", "TeX", "LaTeX2e", "LaTeX", 0};
 313 int const known_phrase_lengths[] = {3, 5, 7, 0};
 314
 315 // string to store the float type to be able to determine the type of subfloats
 316 string float_type = "";
 317
 318
 319 /// splits "x=z, y=b" into a map and an ordered keyword vector
 320 void split_map(string const & s, map<string, string> & res, vector<string> & keys)
 321 {
 322         vector<string> v;
 323         split(s, v);
 324         res.clear();
 325         keys.resize(v.size());
 326         for (size_t i = 0; i < v.size(); ++i) {
 327                 size_t const pos   = v[i].find('=');
 328                 string const index = trimSpaceAndEol(v[i].substr(0, pos));
 329                 string const value = trimSpaceAndEol(v[i].substr(pos + 1, string::npos));
 330                 res[index] = value;
 331                 keys[i] = index;
 332         }
 333 }
 334
 335
 336 /*!
 337  * Split a LaTeX length into value and unit.
 338  * The latter can be a real unit like "pt", or a latex length variable
 339  * like "\textwidth". The unit may contain additional stuff like glue
 340  * lengths, but we don't care, because such lengths are ERT anyway.
 341  * \returns true if \p value and \p unit are valid.
 342  */
 343 bool splitLatexLength(string const & len, string & value, string & unit)
 344 {
 345         if (len.empty())
 346                 return false;
 347         const string::size_type i = len.find_first_not_of(" -+0123456789.,");
 348         //'4,5' is a valid LaTeX length number. Change it to '4.5'
 349         string const length = subst(len, ',', '.');
 350         if (i == string::npos)
 351                 return false;
 352         if (i == 0) {
 353                 if (len[0] == '\\') {
 354                         // We had something like \textwidth without a factor
 355                         value = "1.0";
 356                 } else {
 357                         return false;
 358                 }
 359         } else {
 360                 value = trimSpaceAndEol(string(length, 0, i));
 361         }
 362         if (value == "-")
 363                 value = "-1.0";
 364         // 'cM' is a valid LaTeX length unit. Change it to 'cm'
 365         if (contains(len, '\\'))
 366                 unit = trimSpaceAndEol(string(len, i));
 367         else
 368                 unit = ascii_lowercase(trimSpaceAndEol(string(len, i)));
 369         return true;
 370 }
 371
 372
 373 /// A simple function to translate a latex length to something LyX can
 374 /// understand. Not perfect, but rather best-effort.
 375 bool translate_len(string const & length, string & valstring, string & unit)
 376 {
 377         if (!splitLatexLength(length, valstring, unit))
 378                 return false;
 379         // LyX uses percent values
 380         double value;
 381         istringstream iss(valstring);
 382         iss >> value;
 383         value *= 100;
 384         ostringstream oss;
 385         oss << value;
 386         string const percentval = oss.str();
 387         // a normal length
 388         if (unit.empty() || unit[0] != '\\')
 389                 return true;
 390         string::size_type const i = unit.find(' ');
 391         string const endlen = (i == string::npos) ? string() : string(unit, i);
 392         if (unit == "\\textwidth") {
 393                 valstring = percentval;
 394                 unit = "text%" + endlen;
 395         } else if (unit == "\\columnwidth") {
 396                 valstring = percentval;
 397                 unit = "col%" + endlen;
 398         } else if (unit == "\\paperwidth") {
 399                 valstring = percentval;
 400                 unit = "page%" + endlen;
 401         } else if (unit == "\\linewidth") {
 402                 valstring = percentval;
 403                 unit = "line%" + endlen;
 404         } else if (unit == "\\paperheight") {
 405                 valstring = percentval;
 406                 unit = "pheight%" + endlen;
 407         } else if (unit == "\\textheight") {
 408                 valstring = percentval;
 409                 unit = "theight%" + endlen;
 410         }
 411         return true;
 412 }
 413
 414 }
 415
 416
 417 string translate_len(string const & length)
 418 {
 419         string unit;
 420         string value;
 421         if (translate_len(length, value, unit))
 422                 return value + unit;
 423         // If the input is invalid, return what we have.
 424         return length;
 425 }
 426
 427
 428 namespace {
 429
 430 /*!
 431  * Translates a LaTeX length into \p value, \p unit and
 432  * \p special parts suitable for a box inset.
 433  * The difference from translate_len() is that a box inset knows about
 434  * some special "units" that are stored in \p special.
 435  */
 436 void translate_box_len(string const & length, string & value, string & unit, string & special)
 437 {
 438         if (translate_len(length, value, unit)) {
 439                 if (unit == "\\height" || unit == "\\depth" ||
 440                     unit == "\\totalheight" || unit == "\\width") {
 441                         special = unit.substr(1);
 442                         // The unit is not used, but LyX requires a dummy setting
 443                         unit = "in";
 444                 } else
 445                         special = "none";
 446         } else {
 447                 value.clear();
 448                 unit = length;
 449                 special = "none";
 450         }
 451 }
 452
 453
 454 /*!
 455  * Find a file with basename \p name in path \p path and an extension
 456  * in \p extensions.
 457  */
 458 string find_file(string const & name, string const & path,
 459                  char const * const * extensions)
 460 {
 461         for (char const * const * what = extensions; *what; ++what) {
 462                 string const trial = addExtension(name, *what);
 463                 if (makeAbsPath(trial, path).exists())
 464                         return trial;
 465         }
 466         return string();
 467 }
 468
 469
 470 void begin_inset(ostream & os, string const & name)
 471 {
 472         os << "\n\\begin_inset " << name;
 473 }
 474
 475
 476 void begin_command_inset(ostream & os, string const & name,
 477                          string const & latexname)
 478 {
 479         begin_inset(os, "CommandInset ");
 480         os << name << "\nLatexCommand " << latexname << '\n';
 481 }
 482
 483
 484 void end_inset(ostream & os)
 485 {
 486         os << "\n\\end_inset\n\n";
 487 }
 488
 489
 490 bool skip_braces(Parser & p)
 491 {
 492         if (p.next_token().cat() != catBegin)
 493                 return false;
 494         p.get_token();
 495         if (p.next_token().cat() == catEnd) {
 496                 p.get_token();
 497                 return true;
 498         }
 499         p.putback();
 500         return false;
 501 }
 502
 503
 504 /// replace LaTeX commands in \p s from the unicodesymbols file with their
 505 /// unicode points
 506 docstring convert_unicodesymbols(docstring s)
 507 {
 508         odocstringstream os;
 509         for (size_t i = 0; i < s.size();) {
 510                 if (s[i] != '\\') {
 511                         os.put(s[i++]);
 512                         continue;
 513                 }
 514                 s = s.substr(i);
 515                 bool termination;
 516                 docstring rem;
 517                 set<string> req;
 518                 docstring parsed = encodings.fromLaTeXCommand(s,
 519                                 Encodings::TEXT_CMD, termination, rem, &req);
 520                 set<string>::const_iterator it = req.begin();
 521                 set<string>::const_iterator en = req.end();
 522                 for (; it != en; ++it)
 523                         preamble.registerAutomaticallyLoadedPackage(*it);
 524                 os << parsed;
 525                 s = rem;
 526                 if (s.empty() || s[0] != '\\')
 527                         i = 0;
 528                 else
 529                         i = 1;
 530         }
 531         return os.str();
 532 }
 533
 534
 535 /// try to convert \p s to a valid InsetCommand argument
 536 string convert_command_inset_arg(string s)
 537 {
 538         if (isAscii(s))
 539                 // since we don't know the input encoding we can't use from_utf8
 540                 s = to_utf8(convert_unicodesymbols(from_ascii(s)));
 541         // LyX cannot handle newlines in a latex command
 542         return subst(s, "\n", " ");
 543 }
 544
 545
 546 void handle_backslash(ostream & os, string const & s)
 547 {
 548         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 549                 if (*it == '\\')
 550                         os << "\n\\backslash\n";
 551                 else
 552                         os << *it;
 553         }
 554 }
 555
 556
 557 void handle_ert(ostream & os, string const & s, Context & context)
 558 {
 559         // We must have a valid layout before outputting the ERT inset.
 560         context.check_layout(os);
 561         Context newcontext(true, context.textclass);
 562         begin_inset(os, "ERT");
 563         os << "\nstatus collapsed\n";
 564         newcontext.check_layout(os);
 565         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 566                 if (*it == '\\')
 567                         os << "\n\\backslash\n";
 568                 else if (*it == '\n') {
 569                         newcontext.new_paragraph(os);
 570                         newcontext.check_layout(os);
 571                 } else
 572                         os << *it;
 573         }
 574         newcontext.check_end_layout(os);
 575         end_inset(os);
 576 }
 577
 578
 579 void handle_comment(ostream & os, string const & s, Context & context)
 580 {
 581         // TODO: Handle this better
 582         Context newcontext(true, context.textclass);
 583         begin_inset(os, "ERT");
 584         os << "\nstatus collapsed\n";
 585         newcontext.check_layout(os);
 586         handle_backslash(os, s);
 587         // make sure that our comment is the last thing on the line
 588         newcontext.new_paragraph(os);
 589         newcontext.check_layout(os);
 590         newcontext.check_end_layout(os);
 591         end_inset(os);
 592 }
 593
 594
 595 Layout const * findLayout(TextClass const & textclass, string const & name, bool command)
 596 {
 597         Layout const * layout = findLayoutWithoutModule(textclass, name, command);
 598         if (layout)
 599                 return layout;
 600         if (checkModule(name, command))
 601                 return findLayoutWithoutModule(textclass, name, command);
 602         return layout;
 603 }
 604
 605
 606 InsetLayout const * findInsetLayout(TextClass const & textclass, string const & name, bool command)
 607 {
 608         InsetLayout const * insetlayout = findInsetLayoutWithoutModule(textclass, name, command);
 609         if (insetlayout)
 610                 return insetlayout;
 611         if (checkModule(name, command))
 612                 return findInsetLayoutWithoutModule(textclass, name, command);
 613         return insetlayout;
 614 }
 615
 616
 617 void eat_whitespace(Parser &, ostream &, Context &, bool);
 618
 619
 620 /*!
 621  * Skips whitespace and braces.
 622  * This should be called after a command has been parsed that is not put into
 623  * ERT, and where LyX adds "{}" if needed.
 624  */
 625 void skip_spaces_braces(Parser & p, bool keepws = false)
 626 {
 627         /* The following four examples produce the same typeset output and
 628            should be handled by this function:
 629            - abc \j{} xyz
 630            - abc \j {} xyz
 631            - abc \j
 632              {} xyz
 633            - abc \j %comment
 634              {} xyz
 635          */
 636         // Unfortunately we need to skip comments, too.
 637         // We can't use eat_whitespace since writing them after the {}
 638         // results in different output in some cases.
 639         bool const skipped_spaces = p.skip_spaces(true);
 640         bool const skipped_braces = skip_braces(p);
 641         if (keepws && skipped_spaces && !skipped_braces)
 642                 // put back the space (it is better handled by check_space)
 643                 p.unskip_spaces(true);
 644 }
 645
 646
 647 void output_command_layout(ostream & os, Parser & p, bool outer,
 648                            Context & parent_context,
 649                            Layout const * newlayout)
 650 {
 651         TeXFont const oldFont = parent_context.font;
 652         // save the current font size
 653         string const size = oldFont.size;
 654         // reset the font size to default, because the font size switches
 655         // don't affect section headings and the like
 656         parent_context.font.size = Context::normalfont.size;
 657         // we only need to write the font change if we have an open layout
 658         if (!parent_context.atParagraphStart())
 659                 output_font_change(os, oldFont, parent_context.font);
 660         parent_context.check_end_layout(os);
 661         Context context(true, parent_context.textclass, newlayout,
 662                         parent_context.layout, parent_context.font);
 663         if (parent_context.deeper_paragraph) {
 664                 // We are beginning a nested environment after a
 665                 // deeper paragraph inside the outer list environment.
 666                 // Therefore we don't need to output a "begin deeper".
 667                 context.need_end_deeper = true;
 668         }
 669         context.check_deeper(os);
 670         context.check_layout(os);
 671         unsigned int optargs = 0;
 672         while (optargs < context.layout->optargs) {
 673                 eat_whitespace(p, os, context, false);
 674                 if (p.next_token().cat() == catEscape ||
 675                     p.next_token().character() != '[')
 676                         break;
 677                 p.get_token(); // eat '['
 678                 begin_inset(os, "Argument\n");
 679                 os << "status collapsed\n\n";
 680                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
 681                 end_inset(os);
 682                 eat_whitespace(p, os, context, false);
 683                 ++optargs;
 684         }
 685         unsigned int reqargs = 0;
 686         while (reqargs < context.layout->reqargs) {
 687                 eat_whitespace(p, os, context, false);
 688                 if (p.next_token().cat() != catBegin)
 689                         break;
 690                 p.get_token(); // eat '{'
 691                 begin_inset(os, "Argument\n");
 692                 os << "status collapsed\n\n";
 693                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
 694                 end_inset(os);
 695                 eat_whitespace(p, os, context, false);
 696                 ++reqargs;
 697         }
 698         parse_text(p, os, FLAG_ITEM, outer, context);
 699         context.check_end_layout(os);
 700         if (parent_context.deeper_paragraph) {
 701                 // We must suppress the "end deeper" because we
 702                 // suppressed the "begin deeper" above.
 703                 context.need_end_deeper = false;
 704         }
 705         context.check_end_deeper(os);
 706         // We don't need really a new paragraph, but
 707         // we must make sure that the next item gets a \begin_layout.
 708         parent_context.new_paragraph(os);
 709         // Set the font size to the original value. No need to output it here
 710         // (Context::begin_layout() will do that if needed)
 711         parent_context.font.size = size;
 712 }
 713
 714
 715 /*!
 716  * Output a space if necessary.
 717  * This function gets called for every whitespace token.
 718  *
 719  * We have three cases here:
 720  * 1. A space must be suppressed. Example: The lyxcode case below
 721  * 2. A space may be suppressed. Example: Spaces before "\par"
 722  * 3. A space must not be suppressed. Example: A space between two words
 723  *
 724  * We currently handle only 1. and 3 and from 2. only the case of
 725  * spaces before newlines as a side effect.
 726  *
 727  * 2. could be used to suppress as many spaces as possible. This has two effects:
 728  * - Reimporting LyX generated LaTeX files changes almost no whitespace
 729  * - Superflous whitespace from non LyX generated LaTeX files is removed.
 730  * The drawback is that the logic inside the function becomes
 731  * complicated, and that is the reason why it is not implemented.
 732  */
 733 void check_space(Parser & p, ostream & os, Context & context)
 734 {
 735         Token const next = p.next_token();
 736         Token const curr = p.curr_token();
 737         // A space before a single newline and vice versa must be ignored
 738         // LyX emits a newline before \end{lyxcode}.
 739         // This newline must be ignored,
 740         // otherwise LyX will add an additional protected space.
 741         if (next.cat() == catSpace ||
 742             next.cat() == catNewline ||
 743             (next.cs() == "end" && context.layout->free_spacing && curr.cat() == catNewline)) {
 744                 return;
 745         }
 746         context.check_layout(os);
 747         os << ' ';
 748 }
 749
 750
 751 /*!
 752  * Parse all arguments of \p command
 753  */
 754 void parse_arguments(string const & command,
 755                      vector<ArgumentType> const & template_arguments,
 756                      Parser & p, ostream & os, bool outer, Context & context)
 757 {
 758         string ert = command;
 759         size_t no_arguments = template_arguments.size();
 760         for (size_t i = 0; i < no_arguments; ++i) {
 761                 switch (template_arguments[i]) {
 762                 case required:
 763                 case req_group:
 764                         // This argument contains regular LaTeX
 765                         handle_ert(os, ert + '{', context);
 766                         eat_whitespace(p, os, context, false);
 767                         if (template_arguments[i] == required)
 768                                 parse_text(p, os, FLAG_ITEM, outer, context);
 769                         else
 770                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 771                         ert = "}";
 772                         break;
 773                 case item:
 774                         // This argument consists only of a single item.
 775                         // The presence of '{' or not must be preserved.
 776                         p.skip_spaces();
 777                         if (p.next_token().cat() == catBegin)
 778                                 ert += '{' + p.verbatim_item() + '}';
 779                         else
 780                                 ert += p.verbatim_item();
 781                         break;
 782                 case displaymath:
 783                 case verbatim:
 784                         // This argument may contain special characters
 785                         ert += '{' + p.verbatim_item() + '}';
 786                         break;
 787                 case optional:
 788                 case opt_group:
 789                         // true because we must not eat whitespace
 790                         // if an optional arg follows we must not strip the
 791                         // brackets from this one
 792                         if (i < no_arguments - 1 &&
 793                             template_arguments[i+1] == optional)
 794                                 ert += p.getFullOpt(true);
 795                         else
 796                                 ert += p.getOpt(true);
 797                         break;
 798                 }
 799         }
 800         handle_ert(os, ert, context);
 801 }
 802
 803
 804 /*!
 805  * Check whether \p command is a known command. If yes,
 806  * handle the command with all arguments.
 807  * \return true if the command was parsed, false otherwise.
 808  */
 809 bool parse_command(string const & command, Parser & p, ostream & os,
 810                    bool outer, Context & context)
 811 {
 812         if (known_commands.find(command) != known_commands.end()) {
 813                 parse_arguments(command, known_commands[command], p, os,
 814                                 outer, context);
 815                 return true;
 816         }
 817         return false;
 818 }
 819
 820
 821 /// Parses a minipage or parbox
 822 void parse_box(Parser & p, ostream & os, unsigned outer_flags,
 823                unsigned inner_flags, bool outer, Context & parent_context,
 824                string const & outer_type, string const & special,
 825                string const & inner_type)
 826 {
 827         string position;
 828         string inner_pos;
 829         string hor_pos = "c";
 830         // We need to set the height to the LaTeX default of 1\\totalheight
 831         // for the case when no height argument is given
 832         string height_value = "1";
 833         string height_unit = "in";
 834         string height_special = "totalheight";
 835         string latex_height;
 836         string width_value;
 837         string width_unit;
 838         string latex_width;
 839         string width_special = "none";
 840         if (!inner_type.empty() && p.hasOpt()) {
 841                 if (inner_type != "makebox")
 842                         position = p.getArg('[', ']');
 843                 else {
 844                         latex_width = p.getArg('[', ']');
 845                         translate_box_len(latex_width, width_value, width_unit, width_special);
 846                         position = "t";
 847                 }
 848                 if (position != "t" && position != "c" && position != "b") {
 849                         cerr << "invalid position " << position << " for "
 850                              << inner_type << endl;
 851                         position = "c";
 852                 }
 853                 if (p.hasOpt()) {
 854                         if (inner_type != "makebox") {
 855                                 latex_height = p.getArg('[', ']');
 856                                 translate_box_len(latex_height, height_value, height_unit, height_special);
 857                         } else
 858                                 hor_pos = p.getArg('[', ']');
 859
 860                         if (p.hasOpt()) {
 861                                 inner_pos = p.getArg('[', ']');
 862                                 if (inner_pos != "c" && inner_pos != "t" &&
 863                                     inner_pos != "b" && inner_pos != "s") {
 864                                         cerr << "invalid inner_pos "
 865                                              << inner_pos << " for "
 866                                              << inner_type << endl;
 867                                         inner_pos = position;
 868                                 }
 869                         }
 870                 }
 871         }
 872         if (inner_type.empty()) {
 873                 if (special.empty() && outer_type != "framebox")
 874                         latex_width = "1\\columnwidth";
 875                 else {
 876                         Parser p2(special);
 877                         latex_width = p2.getArg('[', ']');
 878                         string const opt = p2.getArg('[', ']');
 879                         if (!opt.empty()) {
 880                                 hor_pos = opt;
 881                                 if (hor_pos != "l" && hor_pos != "c" &&
 882                                     hor_pos != "r") {
 883                                         cerr << "invalid hor_pos " << hor_pos
 884                                              << " for " << outer_type << endl;
 885                                         hor_pos = "c";
 886                                 }
 887                         }
 888                 }
 889         } else if (inner_type != "makebox")
 890                 latex_width = p.verbatim_item();
 891         // if e.g. only \ovalbox{content} was used, set the width to 1\columnwidth
 892         // as this is LyX's standard for such cases (except for makebox)
 893         // \framebox is more special and handled below
 894         if (latex_width.empty() && inner_type != "makebox"
 895                 && outer_type != "framebox")
 896                 latex_width = "1\\columnwidth";
 897
 898         translate_len(latex_width, width_value, width_unit);
 899
 900         bool shadedparbox = false;
 901         if (inner_type == "shaded") {
 902                 eat_whitespace(p, os, parent_context, false);
 903                 if (outer_type == "parbox") {
 904                         // Eat '{'
 905                         if (p.next_token().cat() == catBegin)
 906                                 p.get_token();
 907                         eat_whitespace(p, os, parent_context, false);
 908                         shadedparbox = true;
 909                 }
 910                 p.get_token();
 911                 p.getArg('{', '}');
 912         }
 913         // If we already read the inner box we have to push the inner env
 914         if (!outer_type.empty() && !inner_type.empty() &&
 915             (inner_flags & FLAG_END))
 916                 active_environments.push_back(inner_type);
 917         // LyX can't handle length variables
 918         bool use_ert = contains(width_unit, '\\') || contains(height_unit, '\\');
 919         if (!use_ert && !outer_type.empty() && !inner_type.empty()) {
 920                 // Look whether there is some content after the end of the
 921                 // inner box, but before the end of the outer box.
 922                 // If yes, we need to output ERT.
 923                 p.pushPosition();
 924                 if (inner_flags & FLAG_END)
 925                         p.verbatimEnvironment(inner_type);
 926                 else
 927                         p.verbatim_item();
 928                 p.skip_spaces(true);
 929                 bool const outer_env(outer_type == "framed" || outer_type == "minipage");
 930                 if ((outer_env && p.next_token().asInput() != "\\end") ||
 931                     (!outer_env && p.next_token().cat() != catEnd)) {
 932                         // something is between the end of the inner box and
 933                         // the end of the outer box, so we need to use ERT.
 934                         use_ert = true;
 935                 }
 936                 p.popPosition();
 937         }
 938         // if only \makebox{content} was used we can set its width to 1\width
 939         // because this identic and also identic to \mbox
 940         // this doesn't work for \framebox{content}, thus we have to use ERT for this
 941         if (latex_width.empty() && inner_type == "makebox") {
 942                 width_value = "1";
 943                 width_unit = "in";
 944                 width_special = "width";
 945         } else if (latex_width.empty() && outer_type == "framebox") {
 946                 use_ert = true;
 947         }
 948         if (use_ert) {
 949                 ostringstream ss;
 950                 if (!outer_type.empty()) {
 951                         if (outer_flags & FLAG_END)
 952                                 ss << "\\begin{" << outer_type << '}';
 953                         else {
 954                                 ss << '\\' << outer_type << '{';
 955                                 if (!special.empty())
 956                                         ss << special;
 957                         }
 958                 }
 959                 if (!inner_type.empty()) {
 960                         if (inner_type != "shaded") {
 961                                 if (inner_flags & FLAG_END)
 962                                         ss << "\\begin{" << inner_type << '}';
 963                                 else
 964                                         ss << '\\' << inner_type;
 965                         }
 966                         if (!position.empty())
 967                                 ss << '[' << position << ']';
 968                         if (!latex_height.empty())
 969                                 ss << '[' << latex_height << ']';
 970                         if (!inner_pos.empty())
 971                                 ss << '[' << inner_pos << ']';
 972                         ss << '{' << latex_width << '}';
 973                         if (!(inner_flags & FLAG_END))
 974                                 ss << '{';
 975                 }
 976                 if (inner_type == "shaded")
 977                         ss << "\\begin{shaded}";
 978                 handle_ert(os, ss.str(), parent_context);
 979                 if (!inner_type.empty()) {
 980                         parse_text(p, os, inner_flags, outer, parent_context);
 981                         if (inner_flags & FLAG_END)
 982                                 handle_ert(os, "\\end{" + inner_type + '}',
 983                                            parent_context);
 984                         else
 985                                 handle_ert(os, "}", parent_context);
 986                 }
 987                 if (!outer_type.empty()) {
 988                         // If we already read the inner box we have to pop
 989                         // the inner env
 990                         if (!inner_type.empty() && (inner_flags & FLAG_END))
 991                                 active_environments.pop_back();
 992
 993                         // Ensure that the end of the outer box is parsed correctly:
 994                         // The opening brace has been eaten by parse_outer_box()
 995                         if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
 996                                 outer_flags &= ~FLAG_ITEM;
 997                                 outer_flags |= FLAG_BRACE_LAST;
 998                         }
 999                         parse_text(p, os, outer_flags, outer, parent_context);
1000                         if (outer_flags & FLAG_END)
1001                                 handle_ert(os, "\\end{" + outer_type + '}',
1002                                            parent_context);
1003                         else if (inner_type.empty() && outer_type == "framebox")
1004                                 // in this case it is already closed later
1005                                 ;
1006                         else
1007                                 handle_ert(os, "}", parent_context);
1008                 }
1009         } else {
1010                 // LyX does not like empty positions, so we have
1011                 // to set them to the LaTeX default values here.
1012                 if (position.empty())
1013                         position = "c";
1014                 if (inner_pos.empty())
1015                         inner_pos = position;
1016                 parent_context.check_layout(os);
1017                 begin_inset(os, "Box ");
1018                 if (outer_type == "framed")
1019                         os << "Framed\n";
1020                 else if (outer_type == "framebox")
1021                         os << "Boxed\n";
1022                 else if (outer_type == "shadowbox")
1023                         os << "Shadowbox\n";
1024                 else if ((outer_type == "shaded" && inner_type.empty()) ||
1025                              (outer_type == "minipage" && inner_type == "shaded") ||
1026                              (outer_type == "parbox" && inner_type == "shaded")) {
1027                         os << "Shaded\n";
1028                         preamble.registerAutomaticallyLoadedPackage("color");
1029                 } else if (outer_type == "doublebox")
1030                         os << "Doublebox\n";
1031                 else if (outer_type.empty())
1032                         os << "Frameless\n";
1033                 else
1034                         os << outer_type << '\n';
1035                 os << "position \"" << position << "\"\n";
1036                 os << "hor_pos \"" << hor_pos << "\"\n";
1037                 os << "has_inner_box " << !inner_type.empty() << "\n";
1038                 os << "inner_pos \"" << inner_pos << "\"\n";
1039                 os << "use_parbox " << (inner_type == "parbox" || shadedparbox)
1040                    << '\n';
1041                 os << "use_makebox " << (inner_type == "makebox") << '\n';
1042                 os << "width \"" << width_value << width_unit << "\"\n";
1043                 os << "special \"" << width_special << "\"\n";
1044                 os << "height \"" << height_value << height_unit << "\"\n";
1045                 os << "height_special \"" << height_special << "\"\n";
1046                 os << "status open\n\n";
1047
1048                 // Unfortunately we can't use parse_text_in_inset:
1049                 // InsetBox::forcePlainLayout() is hard coded and does not
1050                 // use the inset layout. Apart from that do we call parse_text
1051                 // up to two times, but need only one check_end_layout.
1052                 bool const forcePlainLayout =
1053                         (!inner_type.empty() || inner_type == "makebox") &&
1054                         outer_type != "shaded" && outer_type != "framed";
1055                 Context context(true, parent_context.textclass);
1056                 if (forcePlainLayout)
1057                         context.layout = &context.textclass.plainLayout();
1058                 else
1059                         context.font = parent_context.font;
1060
1061                 // If we have no inner box the contents will be read with the outer box
1062                 if (!inner_type.empty())
1063                         parse_text(p, os, inner_flags, outer, context);
1064
1065                 // Ensure that the end of the outer box is parsed correctly:
1066                 // The opening brace has been eaten by parse_outer_box()
1067                 if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
1068                         outer_flags &= ~FLAG_ITEM;
1069                         outer_flags |= FLAG_BRACE_LAST;
1070                 }
1071
1072                 // Find end of outer box, output contents if inner_type is
1073                 // empty and output possible comments
1074                 if (!outer_type.empty()) {
1075                         // If we already read the inner box we have to pop
1076                         // the inner env
1077                         if (!inner_type.empty() && (inner_flags & FLAG_END))
1078                                 active_environments.pop_back();
1079                         // This does not output anything but comments if
1080                         // inner_type is not empty (see use_ert)
1081                         parse_text(p, os, outer_flags, outer, context);
1082                 }
1083
1084                 context.check_end_layout(os);
1085                 end_inset(os);
1086 #ifdef PRESERVE_LAYOUT
1087                 // LyX puts a % after the end of the minipage
1088                 if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) {
1089                         // new paragraph
1090                         //handle_comment(os, "%dummy", parent_context);
1091                         p.get_token();
1092                         p.skip_spaces();
1093                         parent_context.new_paragraph(os);
1094                 }
1095                 else if (p.next_token().cat() == catSpace || p.next_token().cat() == catNewline) {
1096                         //handle_comment(os, "%dummy", parent_context);
1097                         p.get_token();
1098                         p.skip_spaces();
1099                         // We add a protected space if something real follows
1100                         if (p.good() && p.next_token().cat() != catComment) {
1101                                 begin_inset(os, "space ~\n");
1102                                 end_inset(os);
1103                         }
1104                 }
1105 #endif
1106         }
1107 }
1108
1109
1110 void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer,
1111                      Context & parent_context, string const & outer_type,
1112                      string const & special)
1113 {
1114         eat_whitespace(p, os, parent_context, false);
1115         if (flags & FLAG_ITEM) {
1116                 // Eat '{'
1117                 if (p.next_token().cat() == catBegin)
1118                         p.get_token();
1119                 else
1120                         cerr << "Warning: Ignoring missing '{' after \\"
1121                              << outer_type << '.' << endl;
1122                 eat_whitespace(p, os, parent_context, false);
1123         }
1124         string inner;
1125         unsigned int inner_flags = 0;
1126         p.pushPosition();
1127         if (outer_type == "minipage" || outer_type == "parbox") {
1128                 p.skip_spaces(true);
1129                 while (p.hasOpt()) {
1130                         p.getArg('[', ']');
1131                         p.skip_spaces(true);
1132                 }
1133                 p.getArg('{', '}');
1134                 p.skip_spaces(true);
1135                 if (outer_type == "parbox") {
1136                         // Eat '{'
1137                         if (p.next_token().cat() == catBegin)
1138                                 p.get_token();
1139                         p.skip_spaces(true);
1140                 }
1141         }
1142         if (outer_type == "shaded") {
1143                 // These boxes never have an inner box
1144                 ;
1145         } else if (p.next_token().asInput() == "\\parbox") {
1146                 inner = p.get_token().cs();
1147                 inner_flags = FLAG_ITEM;
1148         } else if (p.next_token().asInput() == "\\begin") {
1149                 // Is this a minipage or shaded box?
1150                 p.pushPosition();
1151                 p.get_token();
1152                 inner = p.getArg('{', '}');
1153                 p.popPosition();
1154                 if (inner == "minipage" || inner == "shaded")
1155                         inner_flags = FLAG_END;
1156                 else
1157                         inner = "";
1158         }
1159         p.popPosition();
1160         if (inner_flags == FLAG_END) {
1161                 if (inner != "shaded")
1162                 {
1163                         p.get_token();
1164                         p.getArg('{', '}');
1165                         eat_whitespace(p, os, parent_context, false);
1166                 }
1167                 parse_box(p, os, flags, FLAG_END, outer, parent_context,
1168                           outer_type, special, inner);
1169         } else {
1170                 if (inner_flags == FLAG_ITEM) {
1171                         p.get_token();
1172                         eat_whitespace(p, os, parent_context, false);
1173                 }
1174                 parse_box(p, os, flags, inner_flags, outer, parent_context,
1175                           outer_type, special, inner);
1176         }
1177 }
1178
1179
1180 void parse_listings(Parser & p, ostream & os, Context & parent_context, bool in_line)
1181 {
1182         parent_context.check_layout(os);
1183         begin_inset(os, "listings\n");
1184         if (p.hasOpt()) {
1185                 string arg = p.verbatimOption();
1186                 os << "lstparams " << '"' << arg << '"' << '\n';
1187         }
1188         if (in_line)
1189                 os << "inline true\n";
1190         else
1191                 os << "inline false\n";
1192         os << "status collapsed\n";
1193         Context context(true, parent_context.textclass);
1194         context.layout = &parent_context.textclass.plainLayout();
1195         string s;
1196         if (in_line) {
1197                 s = p.plainCommand('!', '!', "lstinline");
1198                 context.new_paragraph(os);
1199                 context.check_layout(os);
1200         } else
1201                 s = p.plainEnvironment("lstlisting");
1202         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1203                 if (*it == '\\')
1204                         os << "\n\\backslash\n";
1205                 else if (*it == '\n') {
1206                         // avoid adding an empty paragraph at the end
1207                         if (it + 1 != et) {
1208                                 context.new_paragraph(os);
1209                                 context.check_layout(os);
1210                         }
1211                 } else
1212                         os << *it;
1213         }
1214         context.check_end_layout(os);
1215         end_inset(os);
1216 }
1217
1218
1219 /// parse an unknown environment
1220 void parse_unknown_environment(Parser & p, string const & name, ostream & os,
1221                                unsigned flags, bool outer,
1222                                Context & parent_context)
1223 {
1224         if (name == "tabbing")
1225                 // We need to remember that we have to handle '\=' specially
1226                 flags |= FLAG_TABBING;
1227
1228         // We need to translate font changes and paragraphs inside the
1229         // environment to ERT if we have a non standard font.
1230         // Otherwise things like
1231         // \large\begin{foo}\huge bar\end{foo}
1232         // will not work.
1233         bool const specialfont =
1234                 (parent_context.font != parent_context.normalfont);
1235         bool const new_layout_allowed = parent_context.new_layout_allowed;
1236         if (specialfont)
1237                 parent_context.new_layout_allowed = false;
1238         handle_ert(os, "\\begin{" + name + "}", parent_context);
1239         parse_text_snippet(p, os, flags, outer, parent_context);
1240         handle_ert(os, "\\end{" + name + "}", parent_context);
1241         if (specialfont)
1242                 parent_context.new_layout_allowed = new_layout_allowed;
1243 }
1244
1245
1246 void parse_environment(Parser & p, ostream & os, bool outer,
1247                        string & last_env, Context & parent_context)
1248 {
1249         Layout const * newlayout;
1250         InsetLayout const * newinsetlayout = 0;
1251         string const name = p.getArg('{', '}');
1252         const bool is_starred = suffixIs(name, '*');
1253         string const unstarred_name = rtrim(name, "*");
1254         active_environments.push_back(name);
1255
1256         if (is_math_env(name)) {
1257                 parent_context.check_layout(os);
1258                 begin_inset(os, "Formula ");
1259                 os << "\\begin{" << name << "}";
1260                 parse_math(p, os, FLAG_END, MATH_MODE);
1261                 os << "\\end{" << name << "}";
1262                 end_inset(os);
1263                 if (is_display_math_env(name)) {
1264                         // Prevent the conversion of a line break to a space
1265                         // (bug 7668). This does not change the output, but
1266                         // looks ugly in LyX.
1267                         eat_whitespace(p, os, parent_context, false);
1268                 }
1269         }
1270
1271         else if (is_known(name, polyglossia_languages)) {
1272                 // We must begin a new paragraph if not already done
1273                 if (! parent_context.atParagraphStart()) {
1274                         parent_context.check_end_layout(os);
1275                         parent_context.new_paragraph(os);
1276                 }
1277                 // save the language in the context so that it is
1278                 // handled by parse_text
1279                 parent_context.font.language = polyglossia2lyx(name);
1280                 parse_text(p, os, FLAG_END, outer, parent_context);
1281                 // Just in case the environment is empty
1282                 parent_context.extra_stuff.erase();
1283                 // We must begin a new paragraph to reset the language
1284                 parent_context.new_paragraph(os);
1285                 p.skip_spaces();
1286         }
1287
1288         else if (unstarred_name == "tabular" || name == "longtable") {
1289                 eat_whitespace(p, os, parent_context, false);
1290                 string width = "0pt";
1291                 if (name == "tabular*") {
1292                         width = lyx::translate_len(p.getArg('{', '}'));
1293                         eat_whitespace(p, os, parent_context, false);
1294                 }
1295                 parent_context.check_layout(os);
1296                 begin_inset(os, "Tabular ");
1297                 handle_tabular(p, os, name, width, parent_context);
1298                 end_inset(os);
1299                 p.skip_spaces();
1300         }
1301
1302         else if (parent_context.textclass.floats().typeExist(unstarred_name)) {
1303                 eat_whitespace(p, os, parent_context, false);
1304                 string const opt = p.hasOpt() ? p.getArg('[', ']') : string();
1305                 eat_whitespace(p, os, parent_context, false);
1306                 parent_context.check_layout(os);
1307                 begin_inset(os, "Float " + unstarred_name + "\n");
1308                 // store the float type for subfloats
1309                 // subfloats only work with figures and tables
1310                 if (unstarred_name == "figure")
1311                         float_type = unstarred_name;
1312                 else if (unstarred_name == "table")
1313                         float_type = unstarred_name;
1314                 else
1315                         float_type = "";
1316                 if (!opt.empty())
1317                         os << "placement " << opt << '\n';
1318                 if (contains(opt, "H"))
1319                         preamble.registerAutomaticallyLoadedPackage("float");
1320                 else {
1321                         Floating const & fl = parent_context.textclass.floats()
1322                                 .getType(unstarred_name);
1323                         if (!fl.floattype().empty() && fl.usesFloatPkg())
1324                                 preamble.registerAutomaticallyLoadedPackage("float");
1325                 }
1326
1327                 os << "wide " << convert<string>(is_starred)
1328                    << "\nsideways false"
1329                    << "\nstatus open\n\n";
1330                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1331                 end_inset(os);
1332                 // We don't need really a new paragraph, but
1333                 // we must make sure that the next item gets a \begin_layout.
1334                 parent_context.new_paragraph(os);
1335                 p.skip_spaces();
1336                 // the float is parsed thus delete the type
1337                 float_type = "";
1338         }
1339
1340         else if (unstarred_name == "sidewaysfigure"
1341                 || unstarred_name == "sidewaystable") {
1342                 eat_whitespace(p, os, parent_context, false);
1343                 parent_context.check_layout(os);
1344                 if (unstarred_name == "sidewaysfigure")
1345                         begin_inset(os, "Float figure\n");
1346                 else
1347                         begin_inset(os, "Float table\n");
1348                 os << "wide " << convert<string>(is_starred)
1349                    << "\nsideways true"
1350                    << "\nstatus open\n\n";
1351                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1352                 end_inset(os);
1353                 // We don't need really a new paragraph, but
1354                 // we must make sure that the next item gets a \begin_layout.
1355                 parent_context.new_paragraph(os);
1356                 p.skip_spaces();
1357                 preamble.registerAutomaticallyLoadedPackage("rotfloat");
1358         }
1359
1360         else if (name == "wrapfigure" || name == "wraptable") {
1361                 // syntax is \begin{wrapfigure}[lines]{placement}[overhang]{width}
1362                 eat_whitespace(p, os, parent_context, false);
1363                 parent_context.check_layout(os);
1364                 // default values
1365                 string lines = "0";
1366                 string overhang = "0col%";
1367                 // parse
1368                 if (p.hasOpt())
1369                         lines = p.getArg('[', ']');
1370                 string const placement = p.getArg('{', '}');
1371                 if (p.hasOpt())
1372                         overhang = p.getArg('[', ']');
1373                 string const width = p.getArg('{', '}');
1374                 // write
1375                 if (name == "wrapfigure")
1376                         begin_inset(os, "Wrap figure\n");
1377                 else
1378                         begin_inset(os, "Wrap table\n");
1379                 os << "lines " << lines
1380                    << "\nplacement " << placement
1381                    << "\noverhang " << lyx::translate_len(overhang)
1382                    << "\nwidth " << lyx::translate_len(width)
1383                    << "\nstatus open\n\n";
1384                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1385                 end_inset(os);
1386                 // We don't need really a new paragraph, but
1387                 // we must make sure that the next item gets a \begin_layout.
1388                 parent_context.new_paragraph(os);
1389                 p.skip_spaces();
1390                 preamble.registerAutomaticallyLoadedPackage("wrapfig");
1391         }
1392
1393         else if (name == "minipage") {
1394                 eat_whitespace(p, os, parent_context, false);
1395                 // Test whether this is an outer box of a shaded box
1396                 p.pushPosition();
1397                 // swallow arguments
1398                 while (p.hasOpt()) {
1399                         p.getArg('[', ']');
1400                         p.skip_spaces(true);
1401                 }
1402                 p.getArg('{', '}');
1403                 p.skip_spaces(true);
1404                 Token t = p.get_token();
1405                 bool shaded = false;
1406                 if (t.asInput() == "\\begin") {
1407                         p.skip_spaces(true);
1408                         if (p.getArg('{', '}') == "shaded")
1409                                 shaded = true;
1410                 }
1411                 p.popPosition();
1412                 if (shaded)
1413                         parse_outer_box(p, os, FLAG_END, outer,
1414                                         parent_context, name, "shaded");
1415                 else
1416                         parse_box(p, os, 0, FLAG_END, outer, parent_context,
1417                                   "", "", name);
1418                 p.skip_spaces();
1419         }
1420
1421         else if (name == "comment") {
1422                 eat_whitespace(p, os, parent_context, false);
1423                 parent_context.check_layout(os);
1424                 begin_inset(os, "Note Comment\n");
1425                 os << "status open\n";
1426                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1427                 end_inset(os);
1428                 p.skip_spaces();
1429                 skip_braces(p); // eat {} that might by set by LyX behind comments
1430                 preamble.registerAutomaticallyLoadedPackage("verbatim");
1431         }
1432
1433         else if (name == "verbatim") {
1434                 os << "\n\\end_layout\n\n\\begin_layout Verbatim\n";
1435                 string const s = p.plainEnvironment("verbatim");
1436                 string::const_iterator it2 = s.begin();
1437                 for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1438                         if (*it == '\\')
1439                                 os << "\\backslash ";
1440                         else if (*it == '\n') {
1441                                 it2 = it + 1;
1442                                 // avoid adding an empty paragraph at the end
1443                                 // FIXME: if there are 2 consecutive spaces at the end ignore it
1444                                 // because LyX will re-add a \n
1445                                 // This hack must be removed once bug 8049 is fixed!
1446                                 if ((it + 1 != et) && (it + 2 != et || *it2 != '\n'))
1447                                         os << "\n\\end_layout\n\\begin_layout Verbatim\n";
1448                         } else
1449                                 os << *it;
1450                 }
1451                 os << "\n\\end_layout\n\n";
1452                 p.skip_spaces();
1453                 // reset to Standard layout
1454                 os << "\n\\begin_layout Standard\n";
1455         }
1456
1457         else if (name == "CJK") {
1458                 // the scheme is \begin{CJK}{encoding}{mapping}{text}
1459                 // It is impossible to decide if a CJK environment was in its own paragraph or within
1460                 // a line. We therefore always assume a paragraph since the latter is a rare case.
1461                 eat_whitespace(p, os, parent_context, false);
1462                 parent_context.check_end_layout(os);
1463                 // store the encoding to be able to reset it
1464                 string const encoding_old = p.encoding_latex_;
1465                 string const encoding = p.getArg('{', '}');
1466                 // SJIS and BIG5 don't work with LaTeX according to the comment in unicode.cpp
1467                 // JIS does not work with LyX's encoding conversion
1468                 if (encoding != "SJIS" && encoding != "BIG5" && encoding != "JIS")
1469                         p.setEncoding(encoding);
1470                 else
1471                         p.setEncoding("utf8");
1472                 // LyX doesn't support the second argument so if
1473                 // this is used we need to output everything as ERT
1474                 string const mapping = p.getArg('{', '}');
1475                 if ( (!mapping.empty() && mapping != " ")
1476                         || (!is_known(encoding, supported_CJK_encodings))) {
1477                         parent_context.check_layout(os);
1478                         handle_ert(os, "\\begin{" + name + "}{" + encoding + "}{" + mapping + "}",
1479                                        parent_context);
1480                         // we must parse the content as verbatim because e.g. SJIS can contain
1481                         // normally invalid characters
1482                         string const s = p.plainEnvironment("CJK");
1483                         string::const_iterator it2 = s.begin();
1484                         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1485                                 if (*it == '\\')
1486                                         handle_ert(os, "\\", parent_context);
1487                                 else if (*it == '$')
1488                                         handle_ert(os, "$", parent_context);
1489                                 else
1490                                         os << *it;
1491                         }
1492                         p.skip_spaces();
1493                         handle_ert(os, "\\end{" + name + "}",
1494                                        parent_context);
1495                 } else {
1496                         string const lang = CJK2lyx(encoding);
1497                         // store the language because we must reset it at the end
1498                         string const lang_old = parent_context.font.language;
1499                         parent_context.font.language = lang;
1500                         parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1501                         parent_context.font.language = lang_old;
1502                         parent_context.new_paragraph(os);
1503                 }
1504                 p.encoding_latex_ = encoding_old;
1505                 p.skip_spaces();
1506         }
1507
1508         else if (name == "lyxgreyedout") {
1509                 eat_whitespace(p, os, parent_context, false);
1510                 parent_context.check_layout(os);
1511                 begin_inset(os, "Note Greyedout\n");
1512                 os << "status open\n";
1513                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1514                 end_inset(os);
1515                 p.skip_spaces();
1516                 if (!preamble.notefontcolor().empty())
1517                         preamble.registerAutomaticallyLoadedPackage("color");
1518         }
1519
1520         else if (name == "framed" || name == "shaded") {
1521                 eat_whitespace(p, os, parent_context, false);
1522                 parse_outer_box(p, os, FLAG_END, outer, parent_context, name, "");
1523                 p.skip_spaces();
1524         }
1525
1526         else if (name == "lstlisting") {
1527                 eat_whitespace(p, os, parent_context, false);
1528                 // FIXME handle the automatic color package loading
1529                 // uwestoehr asks: In what case color is loaded?
1530                 parse_listings(p, os, parent_context, false);
1531                 p.skip_spaces();
1532         }
1533
1534         else if (!parent_context.new_layout_allowed)
1535                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1536                                           parent_context);
1537
1538         // Alignment and spacing settings
1539         // FIXME (bug xxxx): These settings can span multiple paragraphs and
1540         //                                       therefore are totally broken!
1541         // Note that \centering, raggedright, and raggedleft cannot be handled, as
1542         // they are commands not environments. They are furthermore switches that
1543         // can be ended by another switches, but also by commands like \footnote or
1544         // \parbox. So the only safe way is to leave them untouched.
1545         else if (name == "center" || name == "centering" ||
1546                  name == "flushleft" || name == "flushright" ||
1547                  name == "singlespace" || name == "onehalfspace" ||
1548                  name == "doublespace" || name == "spacing") {
1549                 eat_whitespace(p, os, parent_context, false);
1550                 // We must begin a new paragraph if not already done
1551                 if (! parent_context.atParagraphStart()) {
1552                         parent_context.check_end_layout(os);
1553                         parent_context.new_paragraph(os);
1554                 }
1555                 if (name == "flushleft")
1556                         parent_context.add_extra_stuff("\\align left\n");
1557                 else if (name == "flushright")
1558                         parent_context.add_extra_stuff("\\align right\n");
1559                 else if (name == "center" || name == "centering")
1560                         parent_context.add_extra_stuff("\\align center\n");
1561                 else if (name == "singlespace")
1562                         parent_context.add_extra_stuff("\\paragraph_spacing single\n");
1563                 else if (name == "onehalfspace") {
1564                         parent_context.add_extra_stuff("\\paragraph_spacing onehalf\n");
1565                         preamble.registerAutomaticallyLoadedPackage("setspace");
1566                 } else if (name == "doublespace") {
1567                         parent_context.add_extra_stuff("\\paragraph_spacing double\n");
1568                         preamble.registerAutomaticallyLoadedPackage("setspace");
1569                 } else if (name == "spacing") {
1570                         parent_context.add_extra_stuff("\\paragraph_spacing other " + p.verbatim_item() + "\n");
1571                         preamble.registerAutomaticallyLoadedPackage("setspace");
1572                 }
1573                 parse_text(p, os, FLAG_END, outer, parent_context);
1574                 // Just in case the environment is empty
1575                 parent_context.extra_stuff.erase();
1576                 // We must begin a new paragraph to reset the alignment
1577                 parent_context.new_paragraph(os);
1578                 p.skip_spaces();
1579         }
1580
1581         // The single '=' is meant here.
1582         else if ((newlayout = findLayout(parent_context.textclass, name, false))) {
1583                 eat_whitespace(p, os, parent_context, false);
1584                 Context context(true, parent_context.textclass, newlayout,
1585                                 parent_context.layout, parent_context.font);
1586                 if (parent_context.deeper_paragraph) {
1587                         // We are beginning a nested environment after a
1588                         // deeper paragraph inside the outer list environment.
1589                         // Therefore we don't need to output a "begin deeper".
1590                         context.need_end_deeper = true;
1591                 }
1592                 parent_context.check_end_layout(os);
1593                 if (last_env == name) {
1594                         // we need to output a separator since LyX would export
1595                         // the two environments as one otherwise (bug 5716)
1596                         docstring const sep = from_ascii("--Separator--");
1597                         TeX2LyXDocClass const & textclass(parent_context.textclass);
1598                         if (textclass.hasLayout(sep)) {
1599                                 Context newcontext(parent_context);
1600                                 newcontext.layout = &(textclass[sep]);
1601                                 newcontext.check_layout(os);
1602                                 newcontext.check_end_layout(os);
1603                         } else {
1604                                 parent_context.check_layout(os);
1605                                 begin_inset(os, "Note Note\n");
1606                                 os << "status closed\n";
1607                                 Context newcontext(true, textclass,
1608                                                 &(textclass.defaultLayout()));
1609                                 newcontext.check_layout(os);
1610                                 newcontext.check_end_layout(os);
1611                                 end_inset(os);
1612                                 parent_context.check_end_layout(os);
1613                         }
1614                 }
1615                 switch (context.layout->latextype) {
1616                 case  LATEX_LIST_ENVIRONMENT:
1617                         context.add_par_extra_stuff("\\labelwidthstring "
1618                                                     + p.verbatim_item() + '\n');
1619                         p.skip_spaces();
1620                         break;
1621                 case  LATEX_BIB_ENVIRONMENT:
1622                         p.verbatim_item(); // swallow next arg
1623                         p.skip_spaces();
1624                         break;
1625                 default:
1626                         break;
1627                 }
1628                 context.check_deeper(os);
1629                 // handle known optional and required arguments
1630                 // layouts require all optional arguments before the required ones
1631                 // Unfortunately LyX can't handle arguments of list arguments (bug 7468):
1632                 // It is impossible to place anything after the environment name,
1633                 // but before the first \\item.
1634                 if (context.layout->latextype == LATEX_ENVIRONMENT) {
1635                         bool need_layout = true;
1636                         unsigned int optargs = 0;
1637                         while (optargs < context.layout->optargs) {
1638                                 eat_whitespace(p, os, context, false);
1639                                 if (p.next_token().cat() == catEscape ||
1640                                     p.next_token().character() != '[')
1641                                         break;
1642                                 p.get_token(); // eat '['
1643                                 if (need_layout) {
1644                                         context.check_layout(os);
1645                                         need_layout = false;
1646                                 }
1647                                 begin_inset(os, "Argument\n");
1648                                 os << "status collapsed\n\n";
1649                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
1650                                 end_inset(os);
1651                                 eat_whitespace(p, os, context, false);
1652                                 ++optargs;
1653                         }
1654                         unsigned int reqargs = 0;
1655                         while (reqargs < context.layout->reqargs) {
1656                                 eat_whitespace(p, os, context, false);
1657                                 if (p.next_token().cat() != catBegin)
1658                                         break;
1659                                 p.get_token(); // eat '{'
1660                                 if (need_layout) {
1661                                         context.check_layout(os);
1662                                         need_layout = false;
1663                                 }
1664                                 begin_inset(os, "Argument\n");
1665                                 os << "status collapsed\n\n";
1666                                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
1667                                 end_inset(os);
1668                                 eat_whitespace(p, os, context, false);
1669                                 ++reqargs;
1670                         }
1671                 }
1672                 parse_text(p, os, FLAG_END, outer, context);
1673                 context.check_end_layout(os);
1674                 if (parent_context.deeper_paragraph) {
1675                         // We must suppress the "end deeper" because we
1676                         // suppressed the "begin deeper" above.
1677                         context.need_end_deeper = false;
1678                 }
1679                 context.check_end_deeper(os);
1680                 parent_context.new_paragraph(os);
1681                 p.skip_spaces();
1682                 if (!preamble.titleLayoutFound())
1683                         preamble.titleLayoutFound(newlayout->intitle);
1684                 set<string> const & req = newlayout->requires();
1685                 set<string>::const_iterator it = req.begin();
1686                 set<string>::const_iterator en = req.end();
1687                 for (; it != en; ++it)
1688                         preamble.registerAutomaticallyLoadedPackage(*it);
1689         }
1690
1691         // The single '=' is meant here.
1692         else if ((newinsetlayout = findInsetLayout(parent_context.textclass, name, false))) {
1693                 eat_whitespace(p, os, parent_context, false);
1694                 parent_context.check_layout(os);
1695                 begin_inset(os, "Flex ");
1696                 os << to_utf8(newinsetlayout->name()) << '\n'
1697                    << "status collapsed\n";
1698                 parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
1699                 end_inset(os);
1700         }
1701
1702         else if (name == "appendix") {
1703                 // This is no good latex style, but it works and is used in some documents...
1704                 eat_whitespace(p, os, parent_context, false);
1705                 parent_context.check_end_layout(os);
1706                 Context context(true, parent_context.textclass, parent_context.layout,
1707                                 parent_context.layout, parent_context.font);
1708                 context.check_layout(os);
1709                 os << "\\start_of_appendix\n";
1710                 parse_text(p, os, FLAG_END, outer, context);
1711                 context.check_end_layout(os);
1712                 p.skip_spaces();
1713         }
1714
1715         else if (known_environments.find(name) != known_environments.end()) {
1716                 vector<ArgumentType> arguments = known_environments[name];
1717                 // The last "argument" denotes wether we may translate the
1718                 // environment contents to LyX
1719                 // The default required if no argument is given makes us
1720                 // compatible with the reLyXre environment.
1721                 ArgumentType contents = arguments.empty() ?
1722                         required :
1723                         arguments.back();
1724                 if (!arguments.empty())
1725                         arguments.pop_back();
1726                 // See comment in parse_unknown_environment()
1727                 bool const specialfont =
1728                         (parent_context.font != parent_context.normalfont);
1729                 bool const new_layout_allowed =
1730                         parent_context.new_layout_allowed;
1731                 if (specialfont)
1732                         parent_context.new_layout_allowed = false;
1733                 parse_arguments("\\begin{" + name + "}", arguments, p, os,
1734                                 outer, parent_context);
1735                 if (contents == verbatim)
1736                         handle_ert(os, p.verbatimEnvironment(name),
1737                                    parent_context);
1738                 else
1739                         parse_text_snippet(p, os, FLAG_END, outer,
1740                                            parent_context);
1741                 handle_ert(os, "\\end{" + name + "}", parent_context);
1742                 if (specialfont)
1743                         parent_context.new_layout_allowed = new_layout_allowed;
1744         }
1745
1746         else
1747                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1748                                           parent_context);
1749
1750         last_env = name;
1751         active_environments.pop_back();
1752 }
1753
1754
1755 /// parses a comment and outputs it to \p os.
1756 void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
1757 {
1758         LASSERT(t.cat() == catComment, return);
1759         if (!t.cs().empty()) {
1760                 context.check_layout(os);
1761                 handle_comment(os, '%' + t.cs(), context);
1762                 if (p.next_token().cat() == catNewline) {
1763                         // A newline after a comment line starts a new
1764                         // paragraph
1765                         if (context.new_layout_allowed) {
1766                                 if(!context.atParagraphStart())
1767                                         // Only start a new paragraph if not already
1768                                         // done (we might get called recursively)
1769                                         context.new_paragraph(os);
1770                         } else
1771                                 handle_ert(os, "\n", context);
1772                         eat_whitespace(p, os, context, true);
1773                 }
1774         } else {
1775                 // "%\n" combination
1776                 p.skip_spaces();
1777         }
1778 }
1779
1780
1781 /*!
1782  * Reads spaces and comments until the first non-space, non-comment token.
1783  * New paragraphs (double newlines or \\par) are handled like simple spaces
1784  * if \p eatParagraph is true.
1785  * Spaces are skipped, but comments are written to \p os.
1786  */
1787 void eat_whitespace(Parser & p, ostream & os, Context & context,
1788                     bool eatParagraph)
1789 {
1790         while (p.good()) {
1791                 Token const & t = p.get_token();
1792                 if (t.cat() == catComment)
1793                         parse_comment(p, os, t, context);
1794                 else if ((! eatParagraph && p.isParagraph()) ||
1795                          (t.cat() != catSpace && t.cat() != catNewline)) {
1796                         p.putback();
1797                         return;
1798                 }
1799         }
1800 }
1801
1802
1803 /*!
1804  * Set a font attribute, parse text and reset the font attribute.
1805  * \param attribute Attribute name (e.g. \\family, \\shape etc.)
1806  * \param currentvalue Current value of the attribute. Is set to the new
1807  * value during parsing.
1808  * \param newvalue New value of the attribute
1809  */
1810 void parse_text_attributes(Parser & p, ostream & os, unsigned flags, bool outer,
1811                            Context & context, string const & attribute,
1812                            string & currentvalue, string const & newvalue)
1813 {
1814         context.check_layout(os);
1815         string const oldvalue = currentvalue;
1816         currentvalue = newvalue;
1817         os << '\n' << attribute << ' ' << newvalue << "\n";
1818         parse_text_snippet(p, os, flags, outer, context);
1819         context.check_layout(os);
1820         os << '\n' << attribute << ' ' << oldvalue << "\n";
1821         currentvalue = oldvalue;
1822 }
1823
1824
1825 /// get the arguments of a natbib or jurabib citation command
1826 void get_cite_arguments(Parser & p, bool natbibOrder,
1827         string & before, string & after)
1828 {
1829         // We need to distinguish "" and "[]", so we can't use p.getOpt().
1830
1831         // text before the citation
1832         before.clear();
1833         // text after the citation
1834         after = p.getFullOpt();
1835
1836         if (!after.empty()) {
1837                 before = p.getFullOpt();
1838                 if (natbibOrder && !before.empty())
1839                         swap(before, after);
1840         }
1841 }
1842
1843
1844 /// Convert filenames with TeX macros and/or quotes to something LyX
1845 /// can understand
1846 string const normalize_filename(string const & name)
1847 {
1848         Parser p(trim(name, "\""));
1849         ostringstream os;
1850         while (p.good()) {
1851                 Token const & t = p.get_token();
1852                 if (t.cat() != catEscape)
1853                         os << t.asInput();
1854                 else if (t.cs() == "lyxdot") {
1855                         // This is used by LyX for simple dots in relative
1856                         // names
1857                         os << '.';
1858                         p.skip_spaces();
1859                 } else if (t.cs() == "space") {
1860                         os << ' ';
1861                         p.skip_spaces();
1862                 } else
1863                         os << t.asInput();
1864         }
1865         return os.str();
1866 }
1867
1868
1869 /// Convert \p name from TeX convention (relative to master file) to LyX
1870 /// convention (relative to .lyx file) if it is relative
1871 void fix_relative_filename(string & name)
1872 {
1873         if (FileName::isAbsolute(name))
1874                 return;
1875
1876         name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFileName()),
1877                                    from_utf8(getParentFilePath())));
1878 }
1879
1880
1881 /// Parse a NoWeb Scrap section. The initial "<<" is already parsed.
1882 void parse_noweb(Parser & p, ostream & os, Context & context)
1883 {
1884         // assemble the rest of the keyword
1885         string name("<<");
1886         bool scrap = false;
1887         while (p.good()) {
1888                 Token const & t = p.get_token();
1889                 if (t.asInput() == ">" && p.next_token().asInput() == ">") {
1890                         name += ">>";
1891                         p.get_token();
1892                         scrap = (p.good() && p.next_token().asInput() == "=");
1893                         if (scrap)
1894                                 name += p.get_token().asInput();
1895                         break;
1896                 }
1897                 name += t.asInput();
1898         }
1899
1900         if (!scrap || !context.new_layout_allowed ||
1901             !context.textclass.hasLayout(from_ascii("Scrap"))) {
1902                 cerr << "Warning: Could not interpret '" << name
1903                      << "'. Ignoring it." << endl;
1904                 return;
1905         }
1906
1907         // We use new_paragraph instead of check_end_layout because the stuff
1908         // following the noweb chunk needs to start with a \begin_layout.
1909         // This may create a new paragraph even if there was none in the
1910         // noweb file, but the alternative is an invalid LyX file. Since
1911         // noweb code chunks are implemented with a layout style in LyX they
1912         // always must be in an own paragraph.
1913         context.new_paragraph(os);
1914         Context newcontext(true, context.textclass,
1915                 &context.textclass[from_ascii("Scrap")]);
1916         newcontext.check_layout(os);
1917         os << name;
1918         while (p.good()) {
1919                 Token const & t = p.get_token();
1920                 // We abuse the parser a bit, because this is no TeX syntax
1921                 // at all.
1922                 if (t.cat() == catEscape)
1923                         os << subst(t.asInput(), "\\", "\n\\backslash\n");
1924                 else {
1925                         ostringstream oss;
1926                         Context tmp(false, context.textclass,
1927                                     &context.textclass[from_ascii("Scrap")]);
1928                         tmp.need_end_layout = true;
1929                         tmp.check_layout(oss);
1930                         os << subst(t.asInput(), "\n", oss.str());
1931                 }
1932                 // The scrap chunk is ended by an @ at the beginning of a line.
1933                 // After the @ the line may contain a comment and/or
1934                 // whitespace, but nothing else.
1935                 if (t.asInput() == "@" && p.prev_token().cat() == catNewline &&
1936                     (p.next_token().cat() == catSpace ||
1937                      p.next_token().cat() == catNewline ||
1938                      p.next_token().cat() == catComment)) {
1939                         while (p.good() && p.next_token().cat() == catSpace)
1940                                 os << p.get_token().asInput();
1941                         if (p.next_token().cat() == catComment)
1942                                 // The comment includes a final '\n'
1943                                 os << p.get_token().asInput();
1944                         else {
1945                                 if (p.next_token().cat() == catNewline)
1946                                         p.get_token();
1947                                 os << '\n';
1948                         }
1949                         break;
1950                 }
1951         }
1952         newcontext.check_end_layout(os);
1953 }
1954
1955
1956 /// detects \\def, \\long\\def and \\global\\long\\def with ws and comments
1957 bool is_macro(Parser & p)
1958 {
1959         Token first = p.curr_token();
1960         if (first.cat() != catEscape || !p.good())
1961                 return false;
1962         if (first.cs() == "def")
1963                 return true;
1964         if (first.cs() != "global" && first.cs() != "long")
1965                 return false;
1966         Token second = p.get_token();
1967         int pos = 1;
1968         while (p.good() && !p.isParagraph() && (second.cat() == catSpace ||
1969                second.cat() == catNewline || second.cat() == catComment)) {
1970                 second = p.get_token();
1971                 pos++;
1972         }
1973         bool secondvalid = second.cat() == catEscape;
1974         Token third;
1975         bool thirdvalid = false;
1976         if (p.good() && first.cs() == "global" && secondvalid &&
1977             second.cs() == "long") {
1978                 third = p.get_token();
1979                 pos++;
1980                 while (p.good() && !p.isParagraph() &&
1981                        (third.cat() == catSpace ||
1982                         third.cat() == catNewline ||
1983                         third.cat() == catComment)) {
1984                         third = p.get_token();
1985                         pos++;
1986                 }
1987                 thirdvalid = third.cat() == catEscape;
1988         }
1989         for (int i = 0; i < pos; ++i)
1990                 p.putback();
1991         if (!secondvalid)
1992                 return false;
1993         if (!thirdvalid)
1994                 return (first.cs() == "global" || first.cs() == "long") &&
1995                        second.cs() == "def";
1996         return first.cs() == "global" && second.cs() == "long" &&
1997                third.cs() == "def";
1998 }
1999
2000
2001 /// Parse a macro definition (assumes that is_macro() returned true)
2002 void parse_macro(Parser & p, ostream & os, Context & context)
2003 {
2004         context.check_layout(os);
2005         Token first = p.curr_token();
2006         Token second;
2007         Token third;
2008         string command = first.asInput();
2009         if (first.cs() != "def") {
2010                 p.get_token();
2011                 eat_whitespace(p, os, context, false);
2012                 second = p.curr_token();
2013                 command += second.asInput();
2014                 if (second.cs() != "def") {
2015                         p.get_token();
2016                         eat_whitespace(p, os, context, false);
2017                         third = p.curr_token();
2018                         command += third.asInput();
2019                 }
2020         }
2021         eat_whitespace(p, os, context, false);
2022         string const name = p.get_token().cs();
2023         eat_whitespace(p, os, context, false);
2024
2025         // parameter text
2026         bool simple = true;
2027         string paramtext;
2028         int arity = 0;
2029         while (p.next_token().cat() != catBegin) {
2030                 if (p.next_token().cat() == catParameter) {
2031                         // # found
2032                         p.get_token();
2033                         paramtext += "#";
2034
2035                         // followed by number?
2036                         if (p.next_token().cat() == catOther) {
2037                                 char c = p.getChar();
2038                                 paramtext += c;
2039                                 // number = current arity + 1?
2040                                 if (c == arity + '0' + 1)
2041                                         ++arity;
2042                                 else
2043                                         simple = false;
2044                         } else
2045                                 paramtext += p.get_token().cs();
2046                 } else {
2047                         paramtext += p.get_token().cs();
2048                         simple = false;
2049                 }
2050         }
2051
2052         // only output simple (i.e. compatible) macro as FormulaMacros
2053         string ert = '\\' + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
2054         if (simple) {
2055                 context.check_layout(os);
2056                 begin_inset(os, "FormulaMacro");
2057                 os << "\n\\def" << ert;
2058                 end_inset(os);
2059         } else
2060                 handle_ert(os, command + ert, context);
2061 }
2062
2063
2064 void registerExternalTemplatePackages(string const & name)
2065 {
2066         external::TemplateManager const & etm = external::TemplateManager::get();
2067         external::Template const * const et = etm.getTemplateByName(name);
2068         if (!et)
2069                 return;
2070         external::Template::Formats::const_iterator cit = et->formats.end();
2071         if (pdflatex)
2072                 cit = et->formats.find("PDFLaTeX");
2073         if (cit == et->formats.end())
2074                 // If the template has not specified a PDFLaTeX output,
2075                 // we try the LaTeX format.
2076                 cit = et->formats.find("LaTeX");
2077         if (cit == et->formats.end())
2078                 return;
2079         vector<string>::const_iterator qit = cit->second.requirements.begin();
2080         vector<string>::const_iterator qend = cit->second.requirements.end();
2081         for (; qit != qend; ++qit)
2082                 preamble.registerAutomaticallyLoadedPackage(*qit);
2083 }
2084
2085 } // anonymous namespace
2086
2087
2088 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
2089                 Context & context)
2090 {
2091         Layout const * newlayout = 0;
2092         InsetLayout const * newinsetlayout = 0;
2093         char const * const * where = 0;
2094         // Store the latest bibliographystyle and nocite{*} option
2095         // (needed for bibtex inset)
2096         string btprint;
2097         string bibliographystyle = "default";
2098         bool const use_natbib = preamble.isPackageUsed("natbib");
2099         bool const use_jurabib = preamble.isPackageUsed("jurabib");
2100         string last_env;
2101         while (p.good()) {
2102                 Token const & t = p.get_token();
2103
2104         // it is impossible to determine the correct document language if CJK is used.
2105         // Therefore write a note at the beginning of the document
2106         if (have_CJK) {
2107                 context.check_layout(os);
2108                 begin_inset(os, "Note Note\n");
2109                 os << "status open\n\\begin_layout Plain Layout\n"
2110                    << "\\series bold\n"
2111                    << "Important information:\n"
2112                    << "\\end_layout\n\n"
2113                    << "\\begin_layout Plain Layout\n"
2114                    << "This document contains text in Chinese, Japanese or Korean.\n"
2115                    << " It was therefore impossible for tex2lyx to set the correct document langue for your document."
2116                    << " Please set in the document settings by yourself!\n"
2117                    << "\\end_layout\n";
2118                 end_inset(os);
2119                 have_CJK = false;
2120         }
2121
2122         // it is impossible to determine the correct encoding for non-CJK Japanese.
2123         // Therefore write a note at the beginning of the document
2124         if (is_nonCJKJapanese) {
2125                 context.check_layout(os);
2126                 begin_inset(os, "Note Note\n");
2127                 os << "status open\n\\begin_layout Plain Layout\n"
2128                    << "\\series bold\n"
2129                    << "Important information:\n"
2130                    << "\\end_layout\n\n"
2131                    << "\\begin_layout Plain Layout\n"
2132                    << "This document is in Japanese (non-CJK).\n"
2133                    << " It was therefore impossible for tex2lyx to determine the correct encoding."
2134                    << " The encoding EUC-JP was assumed. If this is incorrect, please set the correct"
2135                    << " encoding in the document settings.\n"
2136                    << "\\end_layout\n";
2137                 end_inset(os);
2138                 is_nonCJKJapanese = false;
2139         }
2140
2141 #ifdef FILEDEBUG
2142                 debugToken(cerr, t, flags);
2143 #endif
2144
2145                 if (flags & FLAG_ITEM) {
2146                         if (t.cat() == catSpace)
2147                                 continue;
2148
2149                         flags &= ~FLAG_ITEM;
2150                         if (t.cat() == catBegin) {
2151                                 // skip the brace and collect everything to the next matching
2152                                 // closing brace
2153                                 flags |= FLAG_BRACE_LAST;
2154                                 continue;
2155                         }
2156
2157                         // handle only this single token, leave the loop if done
2158                         flags |= FLAG_LEAVE;
2159                 }
2160
2161                 if (t.cat() != catEscape && t.character() == ']' &&
2162                     (flags & FLAG_BRACK_LAST))
2163                         return;
2164                 if (t.cat() == catEnd && (flags & FLAG_BRACE_LAST))
2165                         return;
2166
2167                 // If there is anything between \end{env} and \begin{env} we
2168                 // don't need to output a separator.
2169                 if (t.cat() != catSpace && t.cat() != catNewline &&
2170                     t.asInput() != "\\begin")
2171                         last_env = "";
2172
2173                 //
2174                 // cat codes
2175                 //
2176                 if (t.cat() == catMath) {
2177                         // we are inside some text mode thingy, so opening new math is allowed
2178                         context.check_layout(os);
2179                         begin_inset(os, "Formula ");
2180                         Token const & n = p.get_token();
2181                         bool const display(n.cat() == catMath && outer);
2182                         if (display) {
2183                                 // TeX's $$...$$ syntax for displayed math
2184                                 os << "\\[";
2185                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2186                                 os << "\\]";
2187                                 p.get_token(); // skip the second '$' token
2188                         } else {
2189                                 // simple $...$  stuff
2190                                 p.putback();
2191                                 os << '$';
2192                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2193                                 os << '$';
2194                         }
2195                         end_inset(os);
2196                         if (display) {
2197                                 // Prevent the conversion of a line break to a
2198                                 // space (bug 7668). This does not change the
2199                                 // output, but looks ugly in LyX.
2200                                 eat_whitespace(p, os, context, false);
2201                         }
2202                 }
2203
2204                 else if (t.cat() == catSuper || t.cat() == catSub)
2205                         cerr << "catcode " << t << " illegal in text mode\n";
2206
2207                 // Basic support for english quotes. This should be
2208                 // extended to other quotes, but is not so easy (a
2209                 // left english quote is the same as a right german
2210                 // quote...)
2211                 else if (t.asInput() == "`" && p.next_token().asInput() == "`") {
2212                         context.check_layout(os);
2213                         begin_inset(os, "Quotes ");
2214                         os << "eld";
2215                         end_inset(os);
2216                         p.get_token();
2217                         skip_braces(p);
2218                 }
2219                 else if (t.asInput() == "'" && p.next_token().asInput() == "'") {
2220                         context.check_layout(os);
2221                         begin_inset(os, "Quotes ");
2222                         os << "erd";
2223                         end_inset(os);
2224                         p.get_token();
2225                         skip_braces(p);
2226                 }
2227
2228                 else if (t.asInput() == ">" && p.next_token().asInput() == ">") {
2229                         context.check_layout(os);
2230                         begin_inset(os, "Quotes ");
2231                         os << "ald";
2232                         end_inset(os);
2233                         p.get_token();
2234                         skip_braces(p);
2235                 }
2236
2237                 else if (t.asInput() == "<" && p.next_token().asInput() == "<") {
2238                         context.check_layout(os);
2239                         begin_inset(os, "Quotes ");
2240                         os << "ard";
2241                         end_inset(os);
2242                         p.get_token();
2243                         skip_braces(p);
2244                 }
2245
2246                 else if (t.asInput() == "<"
2247                          && p.next_token().asInput() == "<" && noweb_mode) {
2248                         p.get_token();
2249                         parse_noweb(p, os, context);
2250                 }
2251
2252                 else if (t.cat() == catSpace || (t.cat() == catNewline && ! p.isParagraph()))
2253                         check_space(p, os, context);
2254
2255                 else if (t.character() == '[' && noweb_mode &&
2256                          p.next_token().character() == '[') {
2257                         // These can contain underscores
2258                         p.putback();
2259                         string const s = p.getFullOpt() + ']';
2260                         if (p.next_token().character() == ']')
2261                                 p.get_token();
2262                         else
2263                                 cerr << "Warning: Inserting missing ']' in '"
2264                                      << s << "'." << endl;
2265                         handle_ert(os, s, context);
2266                 }
2267
2268                 else if (t.cat() == catLetter) {
2269                         context.check_layout(os);
2270                         // Workaround for bug 4752.
2271                         // FIXME: This whole code block needs to be removed
2272                         //        when the bug is fixed and tex2lyx produces
2273                         //        the updated file format.
2274                         // The replacement algorithm in LyX is so stupid that
2275                         // it even translates a phrase if it is part of a word.
2276                         bool handled = false;
2277                         for (int const * l = known_phrase_lengths; *l; ++l) {
2278                                 string phrase = t.cs();
2279                                 for (int i = 1; i < *l && p.next_token().isAlnumASCII(); ++i)
2280                                         phrase += p.get_token().cs();
2281                                 if (is_known(phrase, known_coded_phrases)) {
2282                                         handle_ert(os, phrase, context);
2283                                         handled = true;
2284                                         break;
2285                                 } else {
2286                                         for (size_t i = 1; i < phrase.length(); ++i)
2287                                                 p.putback();
2288                                 }
2289                         }
2290                         if (!handled)
2291                                 os << t.cs();
2292                 }
2293
2294                 else if (t.cat() == catOther ||
2295                                t.cat() == catAlign ||
2296                                t.cat() == catParameter) {
2297                         // This translates "&" to "\\&" which may be wrong...
2298                         context.check_layout(os);
2299                         os << t.cs();
2300                 }
2301
2302                 else if (p.isParagraph()) {
2303                         if (context.new_layout_allowed)
2304                                 context.new_paragraph(os);
2305                         else
2306                                 handle_ert(os, "\\par ", context);
2307                         eat_whitespace(p, os, context, true);
2308                 }
2309
2310                 else if (t.cat() == catActive) {
2311                         context.check_layout(os);
2312                         if (t.character() == '~') {
2313                                 if (context.layout->free_spacing)
2314                                         os << ' ';
2315                                 else {
2316                                         begin_inset(os, "space ~\n");
2317                                         end_inset(os);
2318                                 }
2319                         } else
2320                                 os << t.cs();
2321                 }
2322
2323                 else if (t.cat() == catBegin) {
2324                         Token const next = p.next_token();
2325                         Token const end = p.next_next_token();
2326                         if (next.cat() == catEnd) {
2327                         // {}
2328                         Token const prev = p.prev_token();
2329                         p.get_token();
2330                         if (p.next_token().character() == '`' ||
2331                             (prev.character() == '-' &&
2332                              p.next_token().character() == '-'))
2333                                 ; // ignore it in {}`` or -{}-
2334                         else
2335                                 handle_ert(os, "{}", context);
2336                         } else if (next.cat() == catEscape &&
2337                                    is_known(next.cs(), known_quotes) &&
2338                                    end.cat() == catEnd) {
2339                                 // Something like {\textquoteright} (e.g.
2340                                 // from writer2latex). LyX writes
2341                                 // \textquoteright{}, so we may skip the
2342                                 // braces here for better readability.
2343                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2344                                                    outer, context);
2345                         } else {
2346                         context.check_layout(os);
2347                         // special handling of font attribute changes
2348                         Token const prev = p.prev_token();
2349                         TeXFont const oldFont = context.font;
2350                         if (next.character() == '[' ||
2351                             next.character() == ']' ||
2352                             next.character() == '*') {
2353                                 p.get_token();
2354                                 if (p.next_token().cat() == catEnd) {
2355                                         os << next.cs();
2356                                         p.get_token();
2357                                 } else {
2358                                         p.putback();
2359                                         handle_ert(os, "{", context);
2360                                         parse_text_snippet(p, os,
2361                                                         FLAG_BRACE_LAST,
2362                                                         outer, context);
2363                                         handle_ert(os, "}", context);
2364                                 }
2365                         } else if (! context.new_layout_allowed) {
2366                                 handle_ert(os, "{", context);
2367                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2368                                                    outer, context);
2369                                 handle_ert(os, "}", context);
2370                         } else if (is_known(next.cs(), known_sizes)) {
2371                                 // next will change the size, so we must
2372                                 // reset it here
2373                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2374                                                    outer, context);
2375                                 if (!context.atParagraphStart())
2376                                         os << "\n\\size "
2377                                            << context.font.size << "\n";
2378                         } else if (is_known(next.cs(), known_font_families)) {
2379                                 // next will change the font family, so we
2380                                 // must reset it here
2381                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2382                                                    outer, context);
2383                                 if (!context.atParagraphStart())
2384                                         os << "\n\\family "
2385                                            << context.font.family << "\n";
2386                         } else if (is_known(next.cs(), known_font_series)) {
2387                                 // next will change the font series, so we
2388                                 // must reset it here
2389                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2390                                                    outer, context);
2391                                 if (!context.atParagraphStart())
2392                                         os << "\n\\series "
2393                                            << context.font.series << "\n";
2394                         } else if (is_known(next.cs(), known_font_shapes)) {
2395                                 // next will change the font shape, so we
2396                                 // must reset it here
2397                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2398                                                    outer, context);
2399                                 if (!context.atParagraphStart())
2400                                         os << "\n\\shape "
2401                                            << context.font.shape << "\n";
2402                         } else if (is_known(next.cs(), known_old_font_families) ||
2403                                    is_known(next.cs(), known_old_font_series) ||
2404                                    is_known(next.cs(), known_old_font_shapes)) {
2405                                 // next will change the font family, series
2406                                 // and shape, so we must reset it here
2407                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2408                                                    outer, context);
2409                                 if (!context.atParagraphStart())
2410                                         os <<  "\n\\family "
2411                                            << context.font.family
2412                                            << "\n\\series "
2413                                            << context.font.series
2414                                            << "\n\\shape "
2415                                            << context.font.shape << "\n";
2416                         } else {
2417                                 handle_ert(os, "{", context);
2418                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2419                                                    outer, context);
2420                                 handle_ert(os, "}", context);
2421                                 }
2422                         }
2423                 }
2424
2425                 else if (t.cat() == catEnd) {
2426                         if (flags & FLAG_BRACE_LAST) {
2427                                 return;
2428                         }
2429                         cerr << "stray '}' in text\n";
2430                         handle_ert(os, "}", context);
2431                 }
2432
2433                 else if (t.cat() == catComment)
2434                         parse_comment(p, os, t, context);
2435
2436                 //
2437                 // control sequences
2438                 //
2439
2440                 else if (t.cs() == "(") {
2441                         context.check_layout(os);
2442                         begin_inset(os, "Formula");
2443                         os << " \\(";
2444                         parse_math(p, os, FLAG_SIMPLE2, MATH_MODE);
2445                         os << "\\)";
2446                         end_inset(os);
2447                 }
2448
2449                 else if (t.cs() == "[") {
2450                         context.check_layout(os);
2451                         begin_inset(os, "Formula");
2452                         os << " \\[";
2453                         parse_math(p, os, FLAG_EQUATION, MATH_MODE);
2454                         os << "\\]";
2455                         end_inset(os);
2456                         // Prevent the conversion of a line break to a space
2457                         // (bug 7668). This does not change the output, but
2458                         // looks ugly in LyX.
2459                         eat_whitespace(p, os, context, false);
2460                 }
2461
2462                 else if (t.cs() == "begin")
2463                         parse_environment(p, os, outer, last_env,
2464                                           context);
2465
2466                 else if (t.cs() == "end") {
2467                         if (flags & FLAG_END) {
2468                                 // eat environment name
2469                                 string const name = p.getArg('{', '}');
2470                                 if (name != active_environment())
2471                                         cerr << "\\end{" + name + "} does not match \\begin{"
2472                                                 + active_environment() + "}\n";
2473                                 return;
2474                         }
2475                         p.error("found 'end' unexpectedly");
2476                 }
2477
2478                 else if (t.cs() == "item") {
2479                         string s;
2480                         bool const optarg = p.hasOpt();
2481                         if (optarg) {
2482                                 // FIXME: This swallows comments, but we cannot use
2483                                 //        eat_whitespace() since we must not output
2484                                 //        anything before the item.
2485                                 p.skip_spaces(true);
2486                                 s = p.verbatimOption();
2487                         } else
2488                                 p.skip_spaces(false);
2489                         context.set_item();
2490                         context.check_layout(os);
2491                         if (context.has_item) {
2492                                 // An item in an unknown list-like environment
2493                                 // FIXME: Do this in check_layout()!
2494                                 context.has_item = false;
2495                                 if (optarg)
2496                                         handle_ert(os, "\\item", context);
2497                                 else
2498                                         handle_ert(os, "\\item ", context);
2499                         }
2500                         if (optarg) {
2501                                 if (context.layout->labeltype != LABEL_MANUAL) {
2502                                         // LyX does not support \item[\mybullet]
2503                                         // in itemize environments
2504                                         Parser p2(s + ']');
2505                                         os << parse_text_snippet(p2,
2506                                                 FLAG_BRACK_LAST, outer, context);
2507                                 } else if (!s.empty()) {
2508                                         // LyX adds braces around the argument,
2509                                         // so we need to remove them here.
2510                                         if (s.size() > 2 && s[0] == '{' &&
2511                                             s[s.size()-1] == '}')
2512                                                 s = s.substr(1, s.size()-2);
2513                                         // If the argument contains a space we
2514                                         // must put it into ERT: Otherwise LyX
2515                                         // would misinterpret the space as
2516                                         // item delimiter (bug 7663)
2517                                         if (contains(s, ' ')) {
2518                                                 handle_ert(os, s, context);
2519                                         } else {
2520                                                 Parser p2(s + ']');
2521                                                 os << parse_text_snippet(p2,
2522                                                         FLAG_BRACK_LAST,
2523                                                         outer, context);
2524                                         }
2525                                         // The space is needed to separate the
2526                                         // item from the rest of the sentence.
2527                                         os << ' ';
2528                                         eat_whitespace(p, os, context, false);
2529                                 }
2530                         }
2531                 }
2532
2533                 else if (t.cs() == "bibitem") {
2534                         context.set_item();
2535                         context.check_layout(os);
2536                         eat_whitespace(p, os, context, false);
2537                         string label = convert_command_inset_arg(p.verbatimOption());
2538                         string key = convert_command_inset_arg(p.verbatim_item());
2539                         if (contains(label, '\\') || contains(key, '\\')) {
2540                                 // LyX can't handle LaTeX commands in labels or keys
2541                                 handle_ert(os, t.asInput() + '[' + label +
2542                                                "]{" + p.verbatim_item() + '}',
2543                                            context);
2544                         } else {
2545                                 begin_command_inset(os, "bibitem", "bibitem");
2546                                 os << "label \"" << label << "\"\n"
2547                                       "key \"" << key << "\"\n";
2548                                 end_inset(os);
2549                         }
2550                 }
2551
2552                 else if (is_macro(p)) {
2553                         // catch the case of \def\inputGnumericTable
2554                         bool macro = true;
2555                         if (t.cs() == "def") {
2556                                 Token second = p.next_token();
2557                                 if (second.cs() == "inputGnumericTable") {
2558                                         p.pushPosition();
2559                                         p.get_token();
2560                                         skip_braces(p);
2561                                         Token third = p.get_token();
2562                                         p.popPosition();
2563                                         if (third.cs() == "input") {
2564                                                 p.get_token();
2565                                                 skip_braces(p);
2566                                                 p.get_token();
2567                                                 string name = normalize_filename(p.verbatim_item());
2568                                                 string const path = getMasterFilePath();
2569                                                 // We want to preserve relative / absolute filenames,
2570                                                 // therefore path is only used for testing
2571                                                 // The file extension is in every case ".tex".
2572                                                 // So we need to remove this extension and check for
2573                                                 // the original one.
2574                                                 name = removeExtension(name);
2575                                                 if (!makeAbsPath(name, path).exists()) {
2576                                                         char const * const Gnumeric_formats[] = {"gnumeric",
2577                                                                 "ods", "xls", 0};
2578                                                         string const Gnumeric_name =
2579                                                                 find_file(name, path, Gnumeric_formats);
2580                                                         if (!Gnumeric_name.empty())
2581                                                                 name = Gnumeric_name;
2582                                                 }
2583                                                 if (makeAbsPath(name, path).exists())
2584                                                         fix_relative_filename(name);
2585                                                 else
2586                                                         cerr << "Warning: Could not find file '"
2587                                                              << name << "'." << endl;
2588                                                 context.check_layout(os);
2589                                                 begin_inset(os, "External\n\ttemplate ");
2590                                                 os << "GnumericSpreadsheet\n\tfilename "
2591                                                    << name << "\n";
2592                                                 end_inset(os);
2593                                                 context.check_layout(os);
2594                                                 macro = false;
2595                                                 // register the packages that are automatically reloaded
2596                                                 // by the Gnumeric template
2597                                                 registerExternalTemplatePackages("GnumericSpreadsheet");
2598                                         }
2599                                 }
2600                         }
2601                         if (macro)
2602                                 parse_macro(p, os, context);
2603                 }
2604
2605                 else if (t.cs() == "noindent") {
2606                         p.skip_spaces();
2607                         context.add_par_extra_stuff("\\noindent\n");
2608                 }
2609
2610                 else if (t.cs() == "appendix") {
2611                         context.add_par_extra_stuff("\\start_of_appendix\n");
2612                         // We need to start a new paragraph. Otherwise the
2613                         // appendix in 'bla\appendix\chapter{' would start
2614                         // too late.
2615                         context.new_paragraph(os);
2616                         // We need to make sure that the paragraph is
2617                         // generated even if it is empty. Otherwise the
2618                         // appendix in '\par\appendix\par\chapter{' would
2619                         // start too late.
2620                         context.check_layout(os);
2621                         // FIXME: This is a hack to prevent paragraph
2622                         // deletion if it is empty. Handle this better!
2623                         handle_comment(os,
2624                                 "%dummy comment inserted by tex2lyx to "
2625                                 "ensure that this paragraph is not empty",
2626                                 context);
2627                         // Both measures above may generate an additional
2628                         // empty paragraph, but that does not hurt, because
2629                         // whitespace does not matter here.
2630                         eat_whitespace(p, os, context, true);
2631                 }
2632
2633                 // Must catch empty dates before findLayout is called below
2634                 else if (t.cs() == "date") {
2635                         eat_whitespace(p, os, context, false);
2636                         p.pushPosition();
2637                         string const date = p.verbatim_item();
2638                         p.popPosition();
2639                         if (date.empty()) {
2640                                 preamble.suppressDate(true);
2641                                 p.verbatim_item();
2642                         } else {
2643                                 preamble.suppressDate(false);
2644                                 if (context.new_layout_allowed &&
2645                                     (newlayout = findLayout(context.textclass,
2646                                                             t.cs(), true))) {
2647                                         // write the layout
2648                                         output_command_layout(os, p, outer,
2649                                                         context, newlayout);
2650                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2651                                         if (!preamble.titleLayoutFound())
2652                                                 preamble.titleLayoutFound(newlayout->intitle);
2653                                         set<string> const & req = newlayout->requires();
2654                                         set<string>::const_iterator it = req.begin();
2655                                         set<string>::const_iterator en = req.end();
2656                                         for (; it != en; ++it)
2657                                                 preamble.registerAutomaticallyLoadedPackage(*it);
2658                                 } else
2659                                         handle_ert(os,
2660                                                 "\\date{" + p.verbatim_item() + '}',
2661                                                 context);
2662                         }
2663                 }
2664
2665                 // Starred section headings
2666                 // Must attempt to parse "Section*" before "Section".
2667                 else if ((p.next_token().asInput() == "*") &&
2668                          context.new_layout_allowed &&
2669                          (newlayout = findLayout(context.textclass, t.cs() + '*', true))) {
2670                         // write the layout
2671                         p.get_token();
2672                         output_command_layout(os, p, outer, context, newlayout);
2673                         p.skip_spaces();
2674                         if (!preamble.titleLayoutFound())
2675                                 preamble.titleLayoutFound(newlayout->intitle);
2676                         set<string> const & req = newlayout->requires();
2677                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2678                                 preamble.registerAutomaticallyLoadedPackage(*it);
2679                 }
2680
2681                 // Section headings and the like
2682                 else if (context.new_layout_allowed &&
2683                          (newlayout = findLayout(context.textclass, t.cs(), true))) {
2684                         // write the layout
2685                         output_command_layout(os, p, outer, context, newlayout);
2686                         p.skip_spaces();
2687                         if (!preamble.titleLayoutFound())
2688                                 preamble.titleLayoutFound(newlayout->intitle);
2689                         set<string> const & req = newlayout->requires();
2690                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2691                                 preamble.registerAutomaticallyLoadedPackage(*it);
2692                 }
2693
2694                 else if (t.cs() == "caption") {
2695                         p.skip_spaces();
2696                         context.check_layout(os);
2697                         p.skip_spaces();
2698                         begin_inset(os, "Caption\n");
2699                         Context newcontext(true, context.textclass);
2700                         newcontext.font = context.font;
2701                         newcontext.check_layout(os);
2702                         if (p.next_token().cat() != catEscape &&
2703                             p.next_token().character() == '[') {
2704                                 p.get_token(); // eat '['
2705                                 begin_inset(os, "Argument\n");
2706                                 os << "status collapsed\n";
2707                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
2708                                 end_inset(os);
2709                                 eat_whitespace(p, os, context, false);
2710                         }
2711                         parse_text(p, os, FLAG_ITEM, outer, context);
2712                         context.check_end_layout(os);
2713                         // We don't need really a new paragraph, but
2714                         // we must make sure that the next item gets a \begin_layout.
2715                         context.new_paragraph(os);
2716                         end_inset(os);
2717                         p.skip_spaces();
2718                         newcontext.check_end_layout(os);
2719                 }
2720
2721                 else if (t.cs() == "subfloat") {
2722                         // the syntax is \subfloat[caption]{content}
2723                         // if it is a table of figure depends on the surrounding float
2724                         bool has_caption = false;
2725                         p.skip_spaces();
2726                         // do nothing if there is no outer float
2727                         if (!float_type.empty()) {
2728                                 context.check_layout(os);
2729                                 p.skip_spaces();
2730                                 begin_inset(os, "Float " + float_type + "\n");
2731                                 os << "wide false"
2732                                    << "\nsideways false"
2733                                    << "\nstatus collapsed\n\n";
2734                                 // test for caption
2735                                 string caption;
2736                                 if (p.next_token().cat() != catEscape &&
2737                                                 p.next_token().character() == '[') {
2738                                                         p.get_token(); // eat '['
2739                                                         caption = parse_text_snippet(p, FLAG_BRACK_LAST, outer, context);
2740                                                         has_caption = true;
2741                                 }
2742                                 // the content
2743                                 parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
2744                                 // the caption comes always as the last
2745                                 if (has_caption) {
2746                                         // we must make sure that the caption gets a \begin_layout
2747                                         os << "\n\\begin_layout Plain Layout";
2748                                         p.skip_spaces();
2749                                         begin_inset(os, "Caption\n");
2750                                         Context newcontext(true, context.textclass);
2751                                         newcontext.font = context.font;
2752                                         newcontext.check_layout(os);
2753                                         os << caption << "\n";
2754                                         newcontext.check_end_layout(os);
2755                                         // We don't need really a new paragraph, but
2756                                         // we must make sure that the next item gets a \begin_layout.
2757                                         //newcontext.new_paragraph(os);
2758                                         end_inset(os);
2759                                         p.skip_spaces();
2760                                 }
2761                                 // We don't need really a new paragraph, but
2762                                 // we must make sure that the next item gets a \begin_layout.
2763                                 if (has_caption)
2764                                         context.new_paragraph(os);
2765                                 end_inset(os);
2766                                 p.skip_spaces();
2767                                 context.check_end_layout(os);
2768                                 // close the layout we opened
2769                                 if (has_caption)
2770                                         os << "\n\\end_layout\n";
2771                         } else {
2772                                 // if the float type is not supported or there is no surrounding float
2773                                 // output it as ERT
2774                                 if (p.hasOpt()) {
2775                                         string opt_arg = convert_command_inset_arg(p.getArg('[', ']'));
2776                                         handle_ert(os, t.asInput() + '[' + opt_arg +
2777                                                "]{" + p.verbatim_item() + '}', context);
2778                                 } else
2779                                         handle_ert(os, t.asInput() + "{" + p.verbatim_item() + '}', context);
2780                         }
2781                 }
2782
2783                 else if (t.cs() == "includegraphics") {
2784                         bool const clip = p.next_token().asInput() == "*";
2785                         if (clip)
2786                                 p.get_token();
2787                         string const arg = p.getArg('[', ']');
2788                         map<string, string> opts;
2789                         vector<string> keys;
2790                         split_map(arg, opts, keys);
2791                         if (clip)
2792                                 opts["clip"] = string();
2793                         string name = normalize_filename(p.verbatim_item());
2794
2795                         string const path = getMasterFilePath();
2796                         // We want to preserve relative / absolute filenames,
2797                         // therefore path is only used for testing
2798                         if (!makeAbsPath(name, path).exists()) {
2799                                 // The file extension is probably missing.
2800                                 // Now try to find it out.
2801                                 string const dvips_name =
2802                                         find_file(name, path,
2803                                                   known_dvips_graphics_formats);
2804                                 string const pdftex_name =
2805                                         find_file(name, path,
2806                                                   known_pdftex_graphics_formats);
2807                                 if (!dvips_name.empty()) {
2808                                         if (!pdftex_name.empty()) {
2809                                                 cerr << "This file contains the "
2810                                                         "latex snippet\n"
2811                                                         "\"\\includegraphics{"
2812                                                      << name << "}\".\n"
2813                                                         "However, files\n\""
2814                                                      << dvips_name << "\" and\n\""
2815                                                      << pdftex_name << "\"\n"
2816                                                         "both exist, so I had to make a "
2817                                                         "choice and took the first one.\n"
2818                                                         "Please move the unwanted one "
2819                                                         "someplace else and try again\n"
2820                                                         "if my choice was wrong."
2821                                                      << endl;
2822                                         }
2823                                         name = dvips_name;
2824                                 } else if (!pdftex_name.empty()) {
2825                                         name = pdftex_name;
2826                                         pdflatex = true;
2827                                 }
2828                         }
2829
2830                         if (makeAbsPath(name, path).exists())
2831                                 fix_relative_filename(name);
2832                         else
2833                                 cerr << "Warning: Could not find graphics file '"
2834                                      << name << "'." << endl;
2835
2836                         context.check_layout(os);
2837                         begin_inset(os, "Graphics ");
2838                         os << "\n\tfilename " << name << '\n';
2839                         if (opts.find("width") != opts.end())
2840                                 os << "\twidth "
2841                                    << translate_len(opts["width"]) << '\n';
2842                         if (opts.find("height") != opts.end())
2843                                 os << "\theight "
2844                                    << translate_len(opts["height"]) << '\n';
2845                         if (opts.find("scale") != opts.end()) {
2846                                 istringstream iss(opts["scale"]);
2847                                 double val;
2848                                 iss >> val;
2849                                 val = val*100;
2850                                 os << "\tscale " << val << '\n';
2851                         }
2852                         if (opts.find("angle") != opts.end()) {
2853                                 os << "\trotateAngle "
2854                                    << opts["angle"] << '\n';
2855                                 vector<string>::const_iterator a =
2856                                         find(keys.begin(), keys.end(), "angle");
2857                                 vector<string>::const_iterator s =
2858                                         find(keys.begin(), keys.end(), "width");
2859                                 if (s == keys.end())
2860                                         s = find(keys.begin(), keys.end(), "height");
2861                                 if (s == keys.end())
2862                                         s = find(keys.begin(), keys.end(), "scale");
2863                                 if (s != keys.end() && distance(s, a) > 0)
2864                                         os << "\tscaleBeforeRotation\n";
2865                         }
2866                         if (opts.find("origin") != opts.end()) {
2867                                 ostringstream ss;
2868                                 string const opt = opts["origin"];
2869                                 if (opt.find('l') != string::npos) ss << "left";
2870                                 if (opt.find('r') != string::npos) ss << "right";
2871                                 if (opt.find('c') != string::npos) ss << "center";
2872                                 if (opt.find('t') != string::npos) ss << "Top";
2873                                 if (opt.find('b') != string::npos) ss << "Bottom";
2874                                 if (opt.find('B') != string::npos) ss << "Baseline";
2875                                 if (!ss.str().empty())
2876                                         os << "\trotateOrigin " << ss.str() << '\n';
2877                                 else
2878                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
2879                         }
2880                         if (opts.find("keepaspectratio") != opts.end())
2881                                 os << "\tkeepAspectRatio\n";
2882                         if (opts.find("clip") != opts.end())
2883                                 os << "\tclip\n";
2884                         if (opts.find("draft") != opts.end())
2885                                 os << "\tdraft\n";
2886                         if (opts.find("bb") != opts.end())
2887                                 os << "\tBoundingBox "
2888                                    << opts["bb"] << '\n';
2889                         int numberOfbbOptions = 0;
2890                         if (opts.find("bbllx") != opts.end())
2891                                 numberOfbbOptions++;
2892                         if (opts.find("bblly") != opts.end())
2893                                 numberOfbbOptions++;
2894                         if (opts.find("bburx") != opts.end())
2895                                 numberOfbbOptions++;
2896                         if (opts.find("bbury") != opts.end())
2897                                 numberOfbbOptions++;
2898                         if (numberOfbbOptions == 4)
2899                                 os << "\tBoundingBox "
2900                                    << opts["bbllx"] << " " << opts["bblly"] << " "
2901                                    << opts["bburx"] << " " << opts["bbury"] << '\n';
2902                         else if (numberOfbbOptions > 0)
2903                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2904                         numberOfbbOptions = 0;
2905                         if (opts.find("natwidth") != opts.end())
2906                                 numberOfbbOptions++;
2907                         if (opts.find("natheight") != opts.end())
2908                                 numberOfbbOptions++;
2909                         if (numberOfbbOptions == 2)
2910                                 os << "\tBoundingBox 0bp 0bp "
2911                                    << opts["natwidth"] << " " << opts["natheight"] << '\n';
2912                         else if (numberOfbbOptions > 0)
2913                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2914                         ostringstream special;
2915                         if (opts.find("hiresbb") != opts.end())
2916                                 special << "hiresbb,";
2917                         if (opts.find("trim") != opts.end())
2918                                 special << "trim,";
2919                         if (opts.find("viewport") != opts.end())
2920                                 special << "viewport=" << opts["viewport"] << ',';
2921                         if (opts.find("totalheight") != opts.end())
2922                                 special << "totalheight=" << opts["totalheight"] << ',';
2923                         if (opts.find("type") != opts.end())
2924                                 special << "type=" << opts["type"] << ',';
2925                         if (opts.find("ext") != opts.end())
2926                                 special << "ext=" << opts["ext"] << ',';
2927                         if (opts.find("read") != opts.end())
2928                                 special << "read=" << opts["read"] << ',';
2929                         if (opts.find("command") != opts.end())
2930                                 special << "command=" << opts["command"] << ',';
2931                         string s_special = special.str();
2932                         if (!s_special.empty()) {
2933                                 // We had special arguments. Remove the trailing ','.
2934                                 os << "\tspecial " << s_special.substr(0, s_special.size() - 1) << '\n';
2935                         }
2936                         // TODO: Handle the unknown settings better.
2937                         // Warn about invalid options.
2938                         // Check whether some option was given twice.
2939                         end_inset(os);
2940                         preamble.registerAutomaticallyLoadedPackage("graphicx");
2941                 }
2942
2943                 else if (t.cs() == "footnote" ||
2944                          (t.cs() == "thanks" && context.layout->intitle)) {
2945                         p.skip_spaces();
2946                         context.check_layout(os);
2947                         begin_inset(os, "Foot\n");
2948                         os << "status collapsed\n\n";
2949                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2950                         end_inset(os);
2951                 }
2952
2953                 else if (t.cs() == "marginpar") {
2954                         p.skip_spaces();
2955                         context.check_layout(os);
2956                         begin_inset(os, "Marginal\n");
2957                         os << "status collapsed\n\n";
2958                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2959                         end_inset(os);
2960                 }
2961
2962                 else if (t.cs() == "lstinline") {
2963                         p.skip_spaces();
2964                         parse_listings(p, os, context, true);
2965                 }
2966
2967                 else if (t.cs() == "ensuremath") {
2968                         p.skip_spaces();
2969                         context.check_layout(os);
2970                         string const s = p.verbatim_item();
2971                         //FIXME: this never triggers in UTF8
2972                         if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5")
2973                                 os << s;
2974                         else
2975                                 handle_ert(os, "\\ensuremath{" + s + "}",
2976                                            context);
2977                 }
2978
2979                 else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
2980                         if (preamble.titleLayoutFound()) {
2981                                 // swallow this
2982                                 skip_spaces_braces(p);
2983                         } else
2984                                 handle_ert(os, t.asInput(), context);
2985                 }
2986
2987                 else if (t.cs() == "tableofcontents" || t.cs() == "lstlistoflistings") {
2988                         context.check_layout(os);
2989                         begin_command_inset(os, "toc", t.cs());
2990                         end_inset(os);
2991                         skip_spaces_braces(p);
2992                         if (t.cs() == "lstlistoflistings")
2993                                 preamble.registerAutomaticallyLoadedPackage("listings");
2994                 }
2995
2996                 else if (t.cs() == "listoffigures") {
2997                         context.check_layout(os);
2998                         begin_inset(os, "FloatList figure\n");
2999                         end_inset(os);
3000                         skip_spaces_braces(p);
3001                 }
3002
3003                 else if (t.cs() == "listoftables") {
3004                         context.check_layout(os);
3005                         begin_inset(os, "FloatList table\n");
3006                         end_inset(os);
3007                         skip_spaces_braces(p);
3008                 }
3009
3010                 else if (t.cs() == "listof") {
3011                         p.skip_spaces(true);
3012                         string const name = p.get_token().cs();
3013                         if (context.textclass.floats().typeExist(name)) {
3014                                 context.check_layout(os);
3015                                 begin_inset(os, "FloatList ");
3016                                 os << name << "\n";
3017                                 end_inset(os);
3018                                 p.get_token(); // swallow second arg
3019                         } else
3020                                 handle_ert(os, "\\listof{" + name + "}", context);
3021                 }
3022
3023                 else if ((where = is_known(t.cs(), known_text_font_families)))
3024                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3025                                 context, "\\family", context.font.family,
3026                                 known_coded_font_families[where - known_text_font_families]);
3027
3028                 else if ((where = is_known(t.cs(), known_text_font_series)))
3029                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3030                                 context, "\\series", context.font.series,
3031                                 known_coded_font_series[where - known_text_font_series]);
3032
3033                 else if ((where = is_known(t.cs(), known_text_font_shapes)))
3034                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3035                                 context, "\\shape", context.font.shape,
3036                                 known_coded_font_shapes[where - known_text_font_shapes]);
3037
3038                 else if (t.cs() == "textnormal" || t.cs() == "normalfont") {
3039                         context.check_layout(os);
3040                         TeXFont oldFont = context.font;
3041                         context.font.init();
3042                         context.font.size = oldFont.size;
3043                         os << "\n\\family " << context.font.family << "\n";
3044                         os << "\n\\series " << context.font.series << "\n";
3045                         os << "\n\\shape " << context.font.shape << "\n";
3046                         if (t.cs() == "textnormal") {
3047                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3048                                 output_font_change(os, context.font, oldFont);
3049                                 context.font = oldFont;
3050                         } else
3051                                 eat_whitespace(p, os, context, false);
3052                 }
3053
3054                 else if (t.cs() == "textcolor") {
3055                         // scheme is \textcolor{color name}{text}
3056                         string const color = p.verbatim_item();
3057                         // we only support the predefined colors of the color package
3058                         if (color == "black" || color == "blue" || color == "cyan"
3059                                 || color == "green" || color == "magenta" || color == "red"
3060                                 || color == "white" || color == "yellow") {
3061                                         context.check_layout(os);
3062                                         os << "\n\\color " << color << "\n";
3063                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3064                                         context.check_layout(os);
3065                                         os << "\n\\color inherit\n";
3066                                         preamble.registerAutomaticallyLoadedPackage("color");
3067                         } else
3068                                 // for custom defined colors
3069                                 handle_ert(os, t.asInput() + "{" + color + "}", context);
3070                 }
3071
3072                 else if (t.cs() == "underbar" || t.cs() == "uline") {
3073                         // \underbar is not 100% correct (LyX outputs \uline
3074                         // of ulem.sty). The difference is that \ulem allows
3075                         // line breaks, and \underbar does not.
3076                         // Do NOT handle \underline.
3077                         // \underbar cuts through y, g, q, p etc.,
3078                         // \underline does not.
3079                         context.check_layout(os);
3080                         os << "\n\\bar under\n";
3081                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3082                         context.check_layout(os);
3083                         os << "\n\\bar default\n";
3084                         preamble.registerAutomaticallyLoadedPackage("ulem");
3085                 }
3086
3087                 else if (t.cs() == "sout") {
3088                         context.check_layout(os);
3089                         os << "\n\\strikeout on\n";
3090                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3091                         context.check_layout(os);
3092                         os << "\n\\strikeout default\n";
3093                         preamble.registerAutomaticallyLoadedPackage("ulem");
3094                 }
3095
3096                 else if (t.cs() == "uuline" || t.cs() == "uwave" ||
3097                          t.cs() == "emph" || t.cs() == "noun") {
3098                         context.check_layout(os);
3099                         os << "\n\\" << t.cs() << " on\n";
3100                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3101                         context.check_layout(os);
3102                         os << "\n\\" << t.cs() << " default\n";
3103                         if (t.cs() == "uuline" || t.cs() == "uwave")
3104                                 preamble.registerAutomaticallyLoadedPackage("ulem");
3105                 }
3106
3107                 else if (t.cs() == "lyxadded" || t.cs() == "lyxdeleted") {
3108                         context.check_layout(os);
3109                         string name = p.getArg('{', '}');
3110                         string localtime = p.getArg('{', '}');
3111                         preamble.registerAuthor(name);
3112                         Author const & author = preamble.getAuthor(name);
3113                         // from_ctime() will fail if LyX decides to output the
3114                         // time in the text language. It might also use a wrong
3115                         // time zone (if the original LyX document was exported
3116                         // with a different time zone).
3117                         time_t ptime = from_ctime(localtime);
3118                         if (ptime == static_cast<time_t>(-1)) {
3119                                 cerr << "Warning: Could not parse time `" << localtime
3120                                      << "´ for change tracking, using current time instead.\n";
3121                                 ptime = current_time();
3122                         }
3123                         if (t.cs() == "lyxadded")
3124                                 os << "\n\\change_inserted ";
3125                         else
3126                                 os << "\n\\change_deleted ";
3127                         os << author.bufferId() << ' ' << ptime << '\n';
3128                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3129                         bool dvipost    = LaTeXPackages::isAvailable("dvipost");
3130                         bool xcolorulem = LaTeXPackages::isAvailable("ulem") &&
3131                                           LaTeXPackages::isAvailable("xcolor");
3132                         // No need to test for luatex, since luatex comes in
3133                         // two flavours (dvi and pdf), like latex, and those
3134                         // are detected by pdflatex.
3135                         if (pdflatex || xetex) {
3136                                 if (xcolorulem) {
3137                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3138                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3139                                         preamble.registerAutomaticallyLoadedPackage("pdfcolmk");
3140                                 }
3141                         } else {
3142                                 if (dvipost) {
3143                                         preamble.registerAutomaticallyLoadedPackage("dvipost");
3144                                 } else if (xcolorulem) {
3145                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3146                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3147                                 }
3148                         }
3149                 }
3150
3151                 else if (t.cs() == "phantom" || t.cs() == "hphantom" ||
3152                              t.cs() == "vphantom") {
3153                         context.check_layout(os);
3154                         if (t.cs() == "phantom")
3155                                 begin_inset(os, "Phantom Phantom\n");
3156                         if (t.cs() == "hphantom")
3157                                 begin_inset(os, "Phantom HPhantom\n");
3158                         if (t.cs() == "vphantom")
3159                                 begin_inset(os, "Phantom VPhantom\n");
3160                         os << "status open\n";
3161                         parse_text_in_inset(p, os, FLAG_ITEM, outer, context,
3162                                             "Phantom");
3163                         end_inset(os);
3164                 }
3165
3166                 else if (t.cs() == "href") {
3167                         context.check_layout(os);
3168                         string target = p.getArg('{', '}');
3169                         string name = p.getArg('{', '}');
3170                         string type;
3171                         size_t i = target.find(':');
3172                         if (i != string::npos) {
3173                                 type = target.substr(0, i + 1);
3174                                 if (type == "mailto:" || type == "file:")
3175                                         target = target.substr(i + 1);
3176                                 // handle the case that name is equal to target, except of "http://"
3177                                 else if (target.substr(i + 3) == name && type == "http:")
3178                                         target = name;
3179                         }
3180                         begin_command_inset(os, "href", "href");
3181                         if (name != target)
3182                                 os << "name \"" << name << "\"\n";
3183                         os << "target \"" << target << "\"\n";
3184                         if (type == "mailto:" || type == "file:")
3185                                 os << "type \"" << type << "\"\n";
3186                         end_inset(os);
3187                         skip_spaces_braces(p);
3188                 }
3189
3190                 else if (t.cs() == "lyxline") {
3191                         // swallow size argument (it is not used anyway)
3192                         p.getArg('{', '}');
3193                         if (!context.atParagraphStart()) {
3194                                 // so our line is in the middle of a paragraph
3195                                 // we need to add a new line, lest this line
3196                                 // follow the other content on that line and
3197                                 // run off the side of the page
3198                                 // FIXME: This may create an empty paragraph,
3199                                 //        but without that it would not be
3200                                 //        possible to set noindent below.
3201                                 //        Fortunately LaTeX does not care
3202                                 //        about the empty paragraph.
3203                                 context.new_paragraph(os);
3204                         }
3205                         if (preamble.indentParagraphs()) {
3206                                 // we need to unindent, lest the line be too long
3207                                 context.add_par_extra_stuff("\\noindent\n");
3208                         }
3209                         context.check_layout(os);
3210                         begin_command_inset(os, "line", "rule");
3211                         os << "offset \"0.5ex\"\n"
3212                               "width \"100line%\"\n"
3213                               "height \"1pt\"\n";
3214                         end_inset(os);
3215                 }
3216
3217                 else if (t.cs() == "rule") {
3218                         string const offset = (p.hasOpt() ? p.getArg('[', ']') : string());
3219                         string const width = p.getArg('{', '}');
3220                         string const thickness = p.getArg('{', '}');
3221                         context.check_layout(os);
3222                         begin_command_inset(os, "line", "rule");
3223                         if (!offset.empty())
3224                                 os << "offset \"" << translate_len(offset) << "\"\n";
3225                         os << "width \"" << translate_len(width) << "\"\n"
3226                                   "height \"" << translate_len(thickness) << "\"\n";
3227                         end_inset(os);
3228                 }
3229
3230                 else if (is_known(t.cs(), known_phrases) ||
3231                          (t.cs() == "protect" &&
3232                           p.next_token().cat() == catEscape &&
3233                           is_known(p.next_token().cs(), known_phrases))) {
3234                         // LyX sometimes puts a \protect in front, so we have to ignore it
3235                         // FIXME: This needs to be changed when bug 4752 is fixed.
3236                         where = is_known(
3237                                 t.cs() == "protect" ? p.get_token().cs() : t.cs(),
3238                                 known_phrases);
3239                         context.check_layout(os);
3240                         os << known_coded_phrases[where - known_phrases];
3241                         skip_spaces_braces(p);
3242                 }
3243
3244                 else if ((where = is_known(t.cs(), known_ref_commands))) {
3245                         string const opt = p.getOpt();
3246                         if (opt.empty()) {
3247                                 context.check_layout(os);
3248                                 begin_command_inset(os, "ref",
3249                                         known_coded_ref_commands[where - known_ref_commands]);
3250                                 os << "reference \""
3251                                    << convert_command_inset_arg(p.verbatim_item())
3252                                    << "\"\n";
3253                                 end_inset(os);
3254                                 if (t.cs() == "vref" || t.cs() == "vpageref")
3255                                         preamble.registerAutomaticallyLoadedPackage("varioref");
3256
3257                         } else {
3258                                 // LyX does not support optional arguments of ref commands
3259                                 handle_ert(os, t.asInput() + '[' + opt + "]{" +
3260                                                p.verbatim_item() + "}", context);
3261                         }
3262                 }
3263
3264                 else if (use_natbib &&
3265                          is_known(t.cs(), known_natbib_commands) &&
3266                          ((t.cs() != "citefullauthor" &&
3267                            t.cs() != "citeyear" &&
3268                            t.cs() != "citeyearpar") ||
3269                           p.next_token().asInput() != "*")) {
3270                         context.check_layout(os);
3271                         string command = t.cs();
3272                         if (p.next_token().asInput() == "*") {
3273                                 command += '*';
3274                                 p.get_token();
3275                         }
3276                         if (command == "citefullauthor")
3277                                 // alternative name for "\\citeauthor*"
3278                                 command = "citeauthor*";
3279
3280                         // text before the citation
3281                         string before;
3282                         // text after the citation
3283                         string after;
3284                         get_cite_arguments(p, true, before, after);
3285
3286                         if (command == "cite") {
3287                                 // \cite without optional argument means
3288                                 // \citet, \cite with at least one optional
3289                                 // argument means \citep.
3290                                 if (before.empty() && after.empty())
3291                                         command = "citet";
3292                                 else
3293                                         command = "citep";
3294                         }
3295                         if (before.empty() && after == "[]")
3296                                 // avoid \citet[]{a}
3297                                 after.erase();
3298                         else if (before == "[]" && after == "[]") {
3299                                 // avoid \citet[][]{a}
3300                                 before.erase();
3301                                 after.erase();
3302                         }
3303                         // remove the brackets around after and before
3304                         if (!after.empty()) {
3305                                 after.erase(0, 1);
3306                                 after.erase(after.length() - 1, 1);
3307                                 after = convert_command_inset_arg(after);
3308                         }
3309                         if (!before.empty()) {
3310                                 before.erase(0, 1);
3311                                 before.erase(before.length() - 1, 1);
3312                                 before = convert_command_inset_arg(before);
3313                         }
3314                         begin_command_inset(os, "citation", command);
3315                         os << "after " << '"' << after << '"' << "\n";
3316                         os << "before " << '"' << before << '"' << "\n";
3317                         os << "key \""
3318                            << convert_command_inset_arg(p.verbatim_item())
3319                            << "\"\n";
3320                         end_inset(os);
3321                 }
3322
3323                 else if (use_jurabib &&
3324                          is_known(t.cs(), known_jurabib_commands) &&
3325                          (t.cs() == "cite" || p.next_token().asInput() != "*")) {
3326                         context.check_layout(os);
3327                         string command = t.cs();
3328                         if (p.next_token().asInput() == "*") {
3329                                 command += '*';
3330                                 p.get_token();
3331                         }
3332                         char argumentOrder = '\0';
3333                         vector<string> const options =
3334                                 preamble.getPackageOptions("jurabib");
3335                         if (find(options.begin(), options.end(),
3336                                       "natbiborder") != options.end())
3337                                 argumentOrder = 'n';
3338                         else if (find(options.begin(), options.end(),
3339                                            "jurabiborder") != options.end())
3340                                 argumentOrder = 'j';
3341
3342                         // text before the citation
3343                         string before;
3344                         // text after the citation
3345                         string after;
3346                         get_cite_arguments(p, argumentOrder != 'j', before, after);
3347
3348                         string const citation = p.verbatim_item();
3349                         if (!before.empty() && argumentOrder == '\0') {
3350                                 cerr << "Warning: Assuming argument order "
3351                                         "of jurabib version 0.6 for\n'"
3352                                      << command << before << after << '{'
3353                                      << citation << "}'.\n"
3354                                         "Add 'jurabiborder' to the jurabib "
3355                                         "package options if you used an\n"
3356                                         "earlier jurabib version." << endl;
3357                         }
3358                         if (!after.empty()) {
3359                                 after.erase(0, 1);
3360                                 after.erase(after.length() - 1, 1);
3361                         }
3362                         if (!before.empty()) {
3363                                 before.erase(0, 1);
3364                                 before.erase(before.length() - 1, 1);
3365                         }
3366                         begin_command_inset(os, "citation", command);
3367                         os << "after " << '"' << after << '"' << "\n";
3368                         os << "before " << '"' << before << '"' << "\n";
3369                         os << "key " << '"' << citation << '"' << "\n";
3370                         end_inset(os);
3371                 }
3372
3373                 else if (t.cs() == "cite"
3374                         || t.cs() == "nocite") {
3375                         context.check_layout(os);
3376                         string after = convert_command_inset_arg(p.getArg('[', ']'));
3377                         string key = convert_command_inset_arg(p.verbatim_item());
3378                         // store the case that it is "\nocite{*}" to use it later for
3379                         // the BibTeX inset
3380                         if (key != "*") {
3381                                 begin_command_inset(os, "citation", t.cs());
3382                                 os << "after " << '"' << after << '"' << "\n";
3383                                 os << "key " << '"' << key << '"' << "\n";
3384                                 end_inset(os);
3385                         } else if (t.cs() == "nocite")
3386                                 btprint = key;
3387                 }
3388
3389                 else if (t.cs() == "index" ||
3390                          (t.cs() == "sindex" && preamble.use_indices() == "true")) {
3391                         context.check_layout(os);
3392                         string const arg = (t.cs() == "sindex" && p.hasOpt()) ?
3393                                 p.getArg('[', ']') : "";
3394                         string const kind = arg.empty() ? "idx" : arg;
3395                         begin_inset(os, "Index ");
3396                         os << kind << "\nstatus collapsed\n";
3397                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, "Index");
3398                         end_inset(os);
3399                         if (kind != "idx")
3400                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3401                 }
3402
3403                 else if (t.cs() == "nomenclature") {
3404                         context.check_layout(os);
3405                         begin_command_inset(os, "nomenclature", "nomenclature");
3406                         string prefix = convert_command_inset_arg(p.getArg('[', ']'));
3407                         if (!prefix.empty())
3408                                 os << "prefix " << '"' << prefix << '"' << "\n";
3409                         os << "symbol " << '"'
3410                            << convert_command_inset_arg(p.verbatim_item());
3411                         os << "\"\ndescription \""
3412                            << convert_command_inset_arg(p.verbatim_item())
3413                            << "\"\n";
3414                         end_inset(os);
3415                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3416                 }
3417
3418                 else if (t.cs() == "label") {
3419                         context.check_layout(os);
3420                         begin_command_inset(os, "label", "label");
3421                         os << "name \""
3422                            << convert_command_inset_arg(p.verbatim_item())
3423                            << "\"\n";
3424                         end_inset(os);
3425                 }
3426
3427                 else if (t.cs() == "printindex") {
3428                         context.check_layout(os);
3429                         begin_command_inset(os, "index_print", "printindex");
3430                         os << "type \"idx\"\n";
3431                         end_inset(os);
3432                         skip_spaces_braces(p);
3433                         preamble.registerAutomaticallyLoadedPackage("makeidx");
3434                         if (preamble.use_indices() == "true")
3435                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3436                 }
3437
3438                 else if (t.cs() == "printnomenclature") {
3439                         string width = "";
3440                         string width_type = "";
3441                         context.check_layout(os);
3442                         begin_command_inset(os, "nomencl_print", "printnomenclature");
3443                         // case of a custom width
3444                         if (p.hasOpt()) {
3445                                 width = p.getArg('[', ']');
3446                                 width = translate_len(width);
3447                                 width_type = "custom";
3448                         }
3449                         // case of no custom width
3450                         // the case of no custom width but the width set
3451                         // via \settowidth{\nomlabelwidth}{***} cannot be supported
3452                         // because the user could have set anything, not only the width
3453                         // of the longest label (which would be width_type = "auto")
3454                         string label = convert_command_inset_arg(p.getArg('{', '}'));
3455                         if (label.empty() && width_type.empty())
3456                                 width_type = "none";
3457                         os << "set_width \"" << width_type << "\"\n";
3458                         if (width_type == "custom")
3459                                 os << "width \"" << width << '\"';
3460                         end_inset(os);
3461                         skip_spaces_braces(p);
3462                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3463                 }
3464
3465                 else if ((t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
3466                         context.check_layout(os);
3467                         begin_inset(os, "script ");
3468                         os << t.cs().substr(4) << '\n';
3469                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
3470                         end_inset(os);
3471                         if (t.cs() == "textsubscript")
3472                                 preamble.registerAutomaticallyLoadedPackage("subscript");
3473                 }
3474
3475                 else if ((where = is_known(t.cs(), known_quotes))) {
3476                         context.check_layout(os);
3477                         begin_inset(os, "Quotes ");
3478                         os << known_coded_quotes[where - known_quotes];
3479                         end_inset(os);
3480                         // LyX adds {} after the quote, so we have to eat
3481                         // spaces here if there are any before a possible
3482                         // {} pair.
3483                         eat_whitespace(p, os, context, false);
3484                         skip_braces(p);
3485                 }
3486
3487                 else if ((where = is_known(t.cs(), known_sizes)) &&
3488                          context.new_layout_allowed) {
3489                         context.check_layout(os);
3490                         TeXFont const oldFont = context.font;
3491                         context.font.size = known_coded_sizes[where - known_sizes];
3492                         output_font_change(os, oldFont, context.font);
3493                         eat_whitespace(p, os, context, false);
3494                 }
3495
3496                 else if ((where = is_known(t.cs(), known_font_families)) &&
3497                          context.new_layout_allowed) {
3498                         context.check_layout(os);
3499                         TeXFont const oldFont = context.font;
3500                         context.font.family =
3501                                 known_coded_font_families[where - known_font_families];
3502                         output_font_change(os, oldFont, context.font);
3503                         eat_whitespace(p, os, context, false);
3504                 }
3505
3506                 else if ((where = is_known(t.cs(), known_font_series)) &&
3507                          context.new_layout_allowed) {
3508                         context.check_layout(os);
3509                         TeXFont const oldFont = context.font;
3510                         context.font.series =
3511                                 known_coded_font_series[where - known_font_series];
3512                         output_font_change(os, oldFont, context.font);
3513                         eat_whitespace(p, os, context, false);
3514                 }
3515
3516                 else if ((where = is_known(t.cs(), known_font_shapes)) &&
3517                          context.new_layout_allowed) {
3518                         context.check_layout(os);
3519                         TeXFont const oldFont = context.font;
3520                         context.font.shape =
3521                                 known_coded_font_shapes[where - known_font_shapes];
3522                         output_font_change(os, oldFont, context.font);
3523                         eat_whitespace(p, os, context, false);
3524                 }
3525                 else if ((where = is_known(t.cs(), known_old_font_families)) &&
3526                          context.new_layout_allowed) {
3527                         context.check_layout(os);
3528                         TeXFont const oldFont = context.font;
3529                         context.font.init();
3530                         context.font.size = oldFont.size;
3531                         context.font.family =
3532                                 known_coded_font_families[where - known_old_font_families];
3533                         output_font_change(os, oldFont, context.font);
3534                         eat_whitespace(p, os, context, false);
3535                 }
3536
3537                 else if ((where = is_known(t.cs(), known_old_font_series)) &&
3538                          context.new_layout_allowed) {
3539                         context.check_layout(os);
3540                         TeXFont const oldFont = context.font;
3541                         context.font.init();
3542                         context.font.size = oldFont.size;
3543                         context.font.series =
3544                                 known_coded_font_series[where - known_old_font_series];
3545                         output_font_change(os, oldFont, context.font);
3546                         eat_whitespace(p, os, context, false);
3547                 }
3548
3549                 else if ((where = is_known(t.cs(), known_old_font_shapes)) &&
3550                          context.new_layout_allowed) {
3551                         context.check_layout(os);
3552                         TeXFont const oldFont = context.font;
3553                         context.font.init();
3554                         context.font.size = oldFont.size;
3555                         context.font.shape =
3556                                 known_coded_font_shapes[where - known_old_font_shapes];
3557                         output_font_change(os, oldFont, context.font);
3558                         eat_whitespace(p, os, context, false);
3559                 }
3560
3561                 else if (t.cs() == "selectlanguage") {
3562                         context.check_layout(os);
3563                         // save the language for the case that a
3564                         // \foreignlanguage is used
3565                         context.font.language = babel2lyx(p.verbatim_item());
3566                         os << "\n\\lang " << context.font.language << "\n";
3567                 }
3568
3569                 else if (t.cs() == "foreignlanguage") {
3570                         string const lang = babel2lyx(p.verbatim_item());
3571                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3572                                               context, "\\lang",
3573                                               context.font.language, lang);
3574                 }
3575
3576                 else if (is_known(t.cs().substr(4, string::npos), polyglossia_languages)) {
3577                         // scheme is \textLANGUAGE{text} where LANGUAGE is in polyglossia_languages[]
3578                         string lang;
3579                         // We have to output the whole command if it has an option
3580                         // because LyX doesn't support this yet, see bug #8214,
3581                         // only if there is a single option specifying a variant, we can handle it.
3582                         if (p.hasOpt()) {
3583                                 string langopts = p.getOpt();
3584                                 // check if the option contains a variant, if yes, extract it
3585                                 string::size_type pos_var = langopts.find("variant");
3586                                 string::size_type i = langopts.find(',');
3587                                 if (pos_var != string::npos){
3588                                         string variant;
3589                                         if (i == string::npos) {
3590                                                 variant = langopts.substr(pos_var + 8, langopts.length() - pos_var - 9);
3591                                                 lang = polyglossia2lyx(variant);
3592                                                 parse_text_attributes(p, os, FLAG_ITEM, outer,
3593                                                                           context, "\\lang",
3594                                                                           context.font.language, lang);
3595                                         }
3596                                         else
3597                                                 handle_ert(os, t.asInput() + langopts, context);
3598                                 } else
3599                                         handle_ert(os, t.asInput() + langopts, context);
3600                         } else {
3601                                 lang = polyglossia2lyx(t.cs().substr(4, string::npos));
3602                                 parse_text_attributes(p, os, FLAG_ITEM, outer,
3603                                                           context, "\\lang",
3604                                                           context.font.language, lang);
3605                         }
3606                 }
3607
3608                 else if (t.cs() == "inputencoding") {
3609                         // nothing to write here
3610                         string const enc = subst(p.verbatim_item(), "\n", " ");
3611                         p.setEncoding(enc);
3612                 }
3613
3614                 else if ((where = is_known(t.cs(), known_special_chars))) {
3615                         context.check_layout(os);
3616                         os << "\\SpecialChar \\"
3617                            << known_coded_special_chars[where - known_special_chars]
3618                            << '\n';
3619                         skip_spaces_braces(p);
3620                 }
3621
3622                 else if (t.cs() == "nobreakdash" && p.next_token().asInput() == "-") {
3623                         context.check_layout(os);
3624                         os << "\\SpecialChar \\nobreakdash-\n";
3625                         p.get_token();
3626                 }
3627
3628                 else if (t.cs() == "textquotedbl") {
3629                         context.check_layout(os);
3630                         os << "\"";
3631                         skip_braces(p);
3632                 }
3633
3634                 else if (t.cs() == "@" && p.next_token().asInput() == ".") {
3635                         context.check_layout(os);
3636                         os << "\\SpecialChar \\@.\n";
3637                         p.get_token();
3638                 }
3639
3640                 else if (t.cs() == "-") {
3641                         context.check_layout(os);
3642                         os << "\\SpecialChar \\-\n";
3643                 }
3644
3645                 else if (t.cs() == "textasciitilde") {
3646                         context.check_layout(os);
3647                         os << '~';
3648                         skip_spaces_braces(p);
3649                 }
3650
3651                 else if (t.cs() == "textasciicircum") {
3652                         context.check_layout(os);
3653                         os << '^';
3654                         skip_spaces_braces(p);
3655                 }
3656
3657                 else if (t.cs() == "textbackslash") {
3658                         context.check_layout(os);
3659                         os << "\n\\backslash\n";
3660                         skip_spaces_braces(p);
3661                 }
3662
3663                 else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#"
3664                             || t.cs() == "$" || t.cs() == "{" || t.cs() == "}"
3665                             || t.cs() == "%") {
3666                         context.check_layout(os);
3667                         os << t.cs();
3668                 }
3669
3670                 else if (t.cs() == "char") {
3671                         context.check_layout(os);
3672                         if (p.next_token().character() == '`') {
3673                                 p.get_token();
3674                                 if (p.next_token().cs() == "\"") {
3675                                         p.get_token();
3676                                         os << '"';
3677                                         skip_braces(p);
3678                                 } else {
3679                                         handle_ert(os, "\\char`", context);
3680                                 }
3681                         } else {
3682                                 handle_ert(os, "\\char", context);
3683                         }
3684                 }
3685
3686                 else if (t.cs() == "verb") {
3687                         context.check_layout(os);
3688                         char const delimiter = p.next_token().character();
3689                         string const arg = p.getArg(delimiter, delimiter);
3690                         ostringstream oss;
3691                         oss << "\\verb" << delimiter << arg << delimiter;
3692                         handle_ert(os, oss.str(), context);
3693                 }
3694
3695                 // Problem: \= creates a tabstop inside the tabbing environment
3696                 // and else an accent. In the latter case we really would want
3697                 // \={o} instead of \= o.
3698                 else if (t.cs() == "=" && (flags & FLAG_TABBING))
3699                         handle_ert(os, t.asInput(), context);
3700
3701                 // accents (see Table 6 in Comprehensive LaTeX Symbol List)
3702                 else if (t.cs().size() == 1
3703                          && contains("\"'.=^`bcdHkrtuv~", t.cs())) {
3704                         context.check_layout(os);
3705                         // try to see whether the string is in unicodesymbols
3706                         bool termination;
3707                         docstring rem;
3708                         string command = t.asInput() + "{"
3709                                 + trimSpaceAndEol(p.verbatim_item())
3710                                 + "}";
3711                         set<string> req;
3712                         docstring s = encodings.fromLaTeXCommand(from_utf8(command),
3713                                 Encodings::TEXT_CMD | Encodings::MATH_CMD,
3714                                 termination, rem, &req);
3715                         if (!s.empty()) {
3716                                 if (!rem.empty())
3717                                         cerr << "When parsing " << command
3718                                              << ", result is " << to_utf8(s)
3719                                              << "+" << to_utf8(rem) << endl;
3720                                 os << to_utf8(s);
3721                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
3722                                         preamble.registerAutomaticallyLoadedPackage(*it);
3723                         } else
3724                                 // we did not find a non-ert version
3725                                 handle_ert(os, command, context);
3726                 }
3727
3728                 else if (t.cs() == "\\") {
3729                         context.check_layout(os);
3730                         if (p.hasOpt())
3731                                 handle_ert(os, "\\\\" + p.getOpt(), context);
3732                         else if (p.next_token().asInput() == "*") {
3733                                 p.get_token();
3734                                 // getOpt() eats the following space if there
3735                                 // is no optional argument, but that is OK
3736                                 // here since it has no effect in the output.
3737                                 handle_ert(os, "\\\\*" + p.getOpt(), context);
3738                         }
3739                         else {
3740                                 begin_inset(os, "Newline newline");
3741                                 end_inset(os);
3742                         }
3743                 }
3744
3745                 else if (t.cs() == "newline" ||
3746                          (t.cs() == "linebreak" && !p.hasOpt())) {
3747                         context.check_layout(os);
3748                         begin_inset(os, "Newline ");
3749                         os << t.cs();
3750                         end_inset(os);
3751                         skip_spaces_braces(p);
3752                 }
3753
3754                 else if (t.cs() == "input" || t.cs() == "include"
3755                          || t.cs() == "verbatiminput") {
3756                         string name = t.cs();
3757                         if (t.cs() == "verbatiminput"
3758                             && p.next_token().asInput() == "*")
3759                                 name += p.get_token().asInput();
3760                         context.check_layout(os);
3761                         string filename(normalize_filename(p.getArg('{', '}')));
3762                         string const path = getMasterFilePath();
3763                         // We want to preserve relative / absolute filenames,
3764                         // therefore path is only used for testing
3765                         if ((t.cs() == "include" || t.cs() == "input") &&
3766                             !makeAbsPath(filename, path).exists()) {
3767                                 // The file extension is probably missing.
3768                                 // Now try to find it out.
3769                                 string const tex_name =
3770                                         find_file(filename, path,
3771                                                   known_tex_extensions);
3772                                 if (!tex_name.empty())
3773                                         filename = tex_name;
3774                         }
3775                         bool external = false;
3776                         string outname;
3777                         if (makeAbsPath(filename, path).exists()) {
3778                                 string const abstexname =
3779                                         makeAbsPath(filename, path).absFileName();
3780                                 string const abslyxname =
3781                                         changeExtension(abstexname, ".lyx");
3782                                 string const absfigname =
3783                                         changeExtension(abstexname, ".fig");
3784                                 fix_relative_filename(filename);
3785                                 string const lyxname =
3786                                         changeExtension(filename, ".lyx");
3787                                 bool xfig = false;
3788                                 external = FileName(absfigname).exists();
3789                                 if (t.cs() == "input") {
3790                                         string const ext = getExtension(abstexname);
3791
3792                                         // Combined PS/LaTeX:
3793                                         // x.eps, x.pstex_t (old xfig)
3794                                         // x.pstex, x.pstex_t (new xfig, e.g. 3.2.5)
3795                                         FileName const absepsname(
3796                                                 changeExtension(abstexname, ".eps"));
3797                                         FileName const abspstexname(
3798                                                 changeExtension(abstexname, ".pstex"));
3799                                         bool const xfigeps =
3800                                                 (absepsname.exists() ||
3801                                                  abspstexname.exists()) &&
3802                                                 ext == "pstex_t";
3803
3804                                         // Combined PDF/LaTeX:
3805                                         // x.pdf, x.pdftex_t (old xfig)
3806                                         // x.pdf, x.pdf_t (new xfig, e.g. 3.2.5)
3807                                         FileName const abspdfname(
3808                                                 changeExtension(abstexname, ".pdf"));
3809                                         bool const xfigpdf =
3810                                                 abspdfname.exists() &&
3811                                                 (ext == "pdftex_t" || ext == "pdf_t");
3812                                         if (xfigpdf)
3813                                                 pdflatex = true;
3814
3815                                         // Combined PS/PDF/LaTeX:
3816                                         // x_pspdftex.eps, x_pspdftex.pdf, x.pspdftex
3817                                         string const absbase2(
3818                                                 removeExtension(abstexname) + "_pspdftex");
3819                                         FileName const abseps2name(
3820                                                 addExtension(absbase2, ".eps"));
3821                                         FileName const abspdf2name(
3822                                                 addExtension(absbase2, ".pdf"));
3823                                         bool const xfigboth =
3824                                                 abspdf2name.exists() &&
3825                                                 abseps2name.exists() && ext == "pspdftex";
3826
3827                                         xfig = xfigpdf || xfigeps || xfigboth;
3828                                         external = external && xfig;
3829                                 }
3830                                 if (external) {
3831                                         outname = changeExtension(filename, ".fig");
3832                                 } else if (xfig) {
3833                                         // Don't try to convert, the result
3834                                         // would be full of ERT.
3835                                         outname = filename;
3836                                 } else if (t.cs() != "verbatiminput" &&
3837                                     tex2lyx(abstexname, FileName(abslyxname),
3838                                             p.getEncoding())) {
3839                                         outname = lyxname;
3840                                 } else {
3841                                         outname = filename;
3842                                 }
3843                         } else {
3844                                 cerr << "Warning: Could not find included file '"
3845                                      << filename << "'." << endl;
3846                                 outname = filename;
3847                         }
3848                         if (external) {
3849                                 begin_inset(os, "External\n");
3850                                 os << "\ttemplate XFig\n"
3851                                    << "\tfilename " << outname << '\n';
3852                                 registerExternalTemplatePackages("XFig");
3853                         } else {
3854                                 begin_command_inset(os, "include", name);
3855                                 os << "preview false\n"
3856                                       "filename \"" << outname << "\"\n";
3857                                 if (t.cs() == "verbatiminput")
3858                                         preamble.registerAutomaticallyLoadedPackage("verbatim");
3859                         }
3860                         end_inset(os);
3861                 }
3862
3863                 else if (t.cs() == "bibliographystyle") {
3864                         // store new bibliographystyle
3865                         bibliographystyle = p.verbatim_item();
3866                         // If any other command than \bibliography and
3867                         // \nocite{*} follows, we need to output the style
3868                         // (because it might be used by that command).
3869                         // Otherwise, it will automatically be output by LyX.
3870                         p.pushPosition();
3871                         bool output = true;
3872                         for (Token t2 = p.get_token(); p.good(); t2 = p.get_token()) {
3873                                 if (t2.cat() == catBegin)
3874                                         break;
3875                                 if (t2.cat() != catEscape)
3876                                         continue;
3877                                 if (t2.cs() == "nocite") {
3878                                         if (p.getArg('{', '}') == "*")
3879                                                 continue;
3880                                 } else if (t2.cs() == "bibliography")
3881                                         output = false;
3882                                 break;
3883                         }
3884                         p.popPosition();
3885                         if (output) {
3886                                 handle_ert(os,
3887                                         "\\bibliographystyle{" + bibliographystyle + '}',
3888                                         context);
3889                         }
3890                 }
3891
3892                 else if (t.cs() == "bibliography") {
3893                         context.check_layout(os);
3894                         begin_command_inset(os, "bibtex", "bibtex");
3895                         if (!btprint.empty()) {
3896                                 os << "btprint " << '"' << "btPrintAll" << '"' << "\n";
3897                                 // clear the string because the next BibTeX inset can be without the
3898                                 // \nocite{*} option
3899                                 btprint.clear();
3900                         }
3901                         os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n";
3902                         // Do we have a bibliographystyle set?
3903                         if (!bibliographystyle.empty())
3904                                 os << "options " << '"' << bibliographystyle << '"' << "\n";
3905                         end_inset(os);
3906                 }
3907
3908                 else if (t.cs() == "parbox") {
3909                         // Test whether this is an outer box of a shaded box
3910                         p.pushPosition();
3911                         // swallow arguments
3912                         while (p.hasOpt()) {
3913                                 p.getArg('[', ']');
3914                                 p.skip_spaces(true);
3915                         }
3916                         p.getArg('{', '}');
3917                         p.skip_spaces(true);
3918                         // eat the '{'
3919                         if (p.next_token().cat() == catBegin)
3920                                 p.get_token();
3921                         p.skip_spaces(true);
3922                         Token to = p.get_token();
3923                         bool shaded = false;
3924                         if (to.asInput() == "\\begin") {
3925                                 p.skip_spaces(true);
3926                                 if (p.getArg('{', '}') == "shaded")
3927                                         shaded = true;
3928                         }
3929                         p.popPosition();
3930                         if (shaded) {
3931                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3932                                                 context, "parbox", "shaded");
3933                         } else
3934                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3935                                           "", "", t.cs());
3936                 }
3937
3938                 else if (t.cs() == "ovalbox" || t.cs() == "Ovalbox" ||
3939                          t.cs() == "shadowbox" || t.cs() == "doublebox")
3940                         parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), "");
3941
3942                 else if (t.cs() == "framebox") {
3943                         if (p.next_token().character() == '(') {
3944                                 //the syntax is: \framebox(x,y)[position]{content}
3945                                 string arg = t.asInput();
3946                                 arg += p.getFullParentheseArg();
3947                                 arg += p.getFullOpt();
3948                                 eat_whitespace(p, os, context, false);
3949                                 handle_ert(os, arg + '{', context);
3950                                 eat_whitespace(p, os, context, false);
3951                                 parse_text(p, os, FLAG_ITEM, outer, context);
3952                                 handle_ert(os, "}", context);
3953                         } else {
3954                                 string special = p.getFullOpt();
3955                                 special += p.getOpt();
3956                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3957                                                 context, t.cs(), special);
3958                         }
3959                 }
3960
3961                 //\makebox() is part of the picture environment and different from \makebox{}
3962                 //\makebox{} will be parsed by parse_box
3963                 else if (t.cs() == "makebox") {
3964                         if (p.next_token().character() == '(') {
3965                                 //the syntax is: \makebox(x,y)[position]{content}
3966                                 string arg = t.asInput();
3967                                 arg += p.getFullParentheseArg();
3968                                 arg += p.getFullOpt();
3969                                 eat_whitespace(p, os, context, false);
3970                                 handle_ert(os, arg + '{', context);
3971                                 eat_whitespace(p, os, context, false);
3972                                 parse_text(p, os, FLAG_ITEM, outer, context);
3973                                 handle_ert(os, "}", context);
3974                         } else
3975                                 //the syntax is: \makebox[width][position]{content}
3976                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3977                                           "", "", t.cs());
3978                 }
3979
3980                 else if (t.cs() == "smallskip" ||
3981                          t.cs() == "medskip" ||
3982                          t.cs() == "bigskip" ||
3983                          t.cs() == "vfill") {
3984                         context.check_layout(os);
3985                         begin_inset(os, "VSpace ");
3986                         os << t.cs();
3987                         end_inset(os);
3988                         skip_spaces_braces(p);
3989                 }
3990
3991                 else if ((where = is_known(t.cs(), known_spaces))) {
3992                         context.check_layout(os);
3993                         begin_inset(os, "space ");
3994                         os << '\\' << known_coded_spaces[where - known_spaces]
3995                            << '\n';
3996                         end_inset(os);
3997                         // LaTeX swallows whitespace after all spaces except
3998                         // "\\,". We have to do that here, too, because LyX
3999                         // adds "{}" which would make the spaces significant.
4000                         if (t.cs() !=  ",")
4001                                 eat_whitespace(p, os, context, false);
4002                         // LyX adds "{}" after all spaces except "\\ " and
4003                         // "\\,", so we have to remove "{}".
4004                         // "\\,{}" is equivalent to "\\," in LaTeX, so we
4005                         // remove the braces after "\\,", too.
4006                         if (t.cs() != " ")
4007                                 skip_braces(p);
4008                 }
4009
4010                 else if (t.cs() == "newpage" ||
4011                          (t.cs() == "pagebreak" && !p.hasOpt()) ||
4012                          t.cs() == "clearpage" ||
4013                          t.cs() == "cleardoublepage") {
4014                         context.check_layout(os);
4015                         begin_inset(os, "Newpage ");
4016                         os << t.cs();
4017                         end_inset(os);
4018                         skip_spaces_braces(p);
4019                 }
4020
4021                 else if (t.cs() == "DeclareRobustCommand" ||
4022                          t.cs() == "DeclareRobustCommandx" ||
4023                          t.cs() == "newcommand" ||
4024                          t.cs() == "newcommandx" ||
4025                          t.cs() == "providecommand" ||
4026                          t.cs() == "providecommandx" ||
4027                          t.cs() == "renewcommand" ||
4028                          t.cs() == "renewcommandx") {
4029                         // DeclareRobustCommand, DeclareRobustCommandx,
4030                         // providecommand and providecommandx could be handled
4031                         // by parse_command(), but we need to call
4032                         // add_known_command() here.
4033                         string name = t.asInput();
4034                         if (p.next_token().asInput() == "*") {
4035                                 // Starred form. Eat '*'
4036                                 p.get_token();
4037                                 name += '*';
4038                         }
4039                         string const command = p.verbatim_item();
4040                         string const opt1 = p.getFullOpt();
4041                         string const opt2 = p.getFullOpt();
4042                         add_known_command(command, opt1, !opt2.empty());
4043                         string const ert = name + '{' + command + '}' +
4044                                            opt1 + opt2 +
4045                                            '{' + p.verbatim_item() + '}';
4046
4047                         if (t.cs() == "DeclareRobustCommand" ||
4048                             t.cs() == "DeclareRobustCommandx" ||
4049                             t.cs() == "providecommand" ||
4050                             t.cs() == "providecommandx" ||
4051                             name[name.length()-1] == '*')
4052                                 handle_ert(os, ert, context);
4053                         else {
4054                                 context.check_layout(os);
4055                                 begin_inset(os, "FormulaMacro");
4056                                 os << "\n" << ert;
4057                                 end_inset(os);
4058                         }
4059                 }
4060
4061                 else if (t.cs() == "let" && p.next_token().asInput() != "*") {
4062                         // let could be handled by parse_command(),
4063                         // but we need to call add_known_command() here.
4064                         string ert = t.asInput();
4065                         string name;
4066                         p.skip_spaces();
4067                         if (p.next_token().cat() == catBegin) {
4068                                 name = p.verbatim_item();
4069                                 ert += '{' + name + '}';
4070                         } else {
4071                                 name = p.verbatim_item();
4072                                 ert += name;
4073                         }
4074                         string command;
4075                         p.skip_spaces();
4076                         if (p.next_token().cat() == catBegin) {
4077                                 command = p.verbatim_item();
4078                                 ert += '{' + command + '}';
4079                         } else {
4080                                 command = p.verbatim_item();
4081                                 ert += command;
4082                         }
4083                         // If command is known, make name known too, to parse
4084                         // its arguments correctly. For this reason we also
4085                         // have commands in syntax.default that are hardcoded.
4086                         CommandMap::iterator it = known_commands.find(command);
4087                         if (it != known_commands.end())
4088                                 known_commands[t.asInput()] = it->second;
4089                         handle_ert(os, ert, context);
4090                 }
4091
4092                 else if (t.cs() == "hspace" || t.cs() == "vspace") {
4093                         bool starred = false;
4094                         if (p.next_token().asInput() == "*") {
4095                                 p.get_token();
4096                                 starred = true;
4097                         }
4098                         string name = t.asInput();
4099                         string const length = p.verbatim_item();
4100                         string unit;
4101                         string valstring;
4102                         bool valid = splitLatexLength(length, valstring, unit);
4103                         bool known_hspace = false;
4104                         bool known_vspace = false;
4105                         bool known_unit = false;
4106                         double value;
4107                         if (valid) {
4108                                 istringstream iss(valstring);
4109                                 iss >> value;
4110                                 if (value == 1.0) {
4111                                         if (t.cs()[0] == 'h') {
4112                                                 if (unit == "\\fill") {
4113                                                         if (!starred) {
4114                                                                 unit = "";
4115                                                                 name = "\\hfill";
4116                                                         }
4117                                                         known_hspace = true;
4118                                                 }
4119                                         } else {
4120                                                 if (unit == "\\smallskipamount") {
4121                                                         unit = "smallskip";
4122                                                         known_vspace = true;
4123                                                 } else if (unit == "\\medskipamount") {
4124                                                         unit = "medskip";
4125                                                         known_vspace = true;
4126                                                 } else if (unit == "\\bigskipamount") {
4127                                                         unit = "bigskip";
4128                                                         known_vspace = true;
4129                                                 } else if (unit == "\\fill") {
4130                                                         unit = "vfill";
4131                                                         known_vspace = true;
4132                                                 }
4133                                         }
4134                                 }
4135                                 if (!known_hspace && !known_vspace) {
4136                                         switch (unitFromString(unit)) {
4137                                         case Length::SP:
4138                                         case Length::PT:
4139                                         case Length::BP:
4140                                         case Length::DD:
4141                                         case Length::MM:
4142                                         case Length::PC:
4143                                         case Length::CC:
4144                                         case Length::CM:
4145                                         case Length::IN:
4146                                         case Length::EX:
4147                                         case Length::EM:
4148                                         case Length::MU:
4149                                                 known_unit = true;
4150                                                 break;
4151                                         default:
4152                                                 break;
4153                                         }
4154                                 }
4155                         }
4156
4157                         if (t.cs()[0] == 'h' && (known_unit || known_hspace)) {
4158                                 // Literal horizontal length or known variable
4159                                 context.check_layout(os);
4160                                 begin_inset(os, "space ");
4161                                 os << name;
4162                                 if (starred)
4163                                         os << '*';
4164                                 os << '{';
4165                                 if (known_hspace)
4166                                         os << unit;
4167                                 os << "}";
4168                                 if (known_unit && !known_hspace)
4169                                         os << "\n\\length "
4170                                            << translate_len(length);
4171                                 end_inset(os);
4172                         } else if (known_unit || known_vspace) {
4173                                 // Literal vertical length or known variable
4174                                 context.check_layout(os);
4175                                 begin_inset(os, "VSpace ");
4176                                 if (known_unit)
4177                                         os << value;
4178                                 os << unit;
4179                                 if (starred)
4180                                         os << '*';
4181                                 end_inset(os);
4182                         } else {
4183                                 // LyX can't handle other length variables in Inset VSpace/space
4184                                 if (starred)
4185                                         name += '*';
4186                                 if (valid) {
4187                                         if (value == 1.0)
4188                                                 handle_ert(os, name + '{' + unit + '}', context);
4189                                         else if (value == -1.0)
4190                                                 handle_ert(os, name + "{-" + unit + '}', context);
4191                                         else
4192                                                 handle_ert(os, name + '{' + valstring + unit + '}', context);
4193                                 } else
4194                                         handle_ert(os, name + '{' + length + '}', context);
4195                         }
4196                 }
4197
4198                 // The single '=' is meant here.
4199                 else if ((newinsetlayout = findInsetLayout(context.textclass, t.cs(), true))) {
4200                         p.skip_spaces();
4201                         context.check_layout(os);
4202                         begin_inset(os, "Flex ");
4203                         os << to_utf8(newinsetlayout->name()) << '\n'
4204                            << "status collapsed\n";
4205                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
4206                         end_inset(os);
4207                 }
4208
4209                 else if (t.cs() == "includepdf") {
4210                         p.skip_spaces();
4211                         string const arg = p.getArg('[', ']');
4212                         map<string, string> opts;
4213                         vector<string> keys;
4214                         split_map(arg, opts, keys);
4215                         string name = normalize_filename(p.verbatim_item());
4216                         string const path = getMasterFilePath();
4217                         // We want to preserve relative / absolute filenames,
4218                         // therefore path is only used for testing
4219                         if (!makeAbsPath(name, path).exists()) {
4220                                 // The file extension is probably missing.
4221                                 // Now try to find it out.
4222                                 char const * const pdfpages_format[] = {"pdf", 0};
4223                                 string const pdftex_name =
4224                                         find_file(name, path, pdfpages_format);
4225                                 if (!pdftex_name.empty()) {
4226                                         name = pdftex_name;
4227                                         pdflatex = true;
4228                                 }
4229                         }
4230                         if (makeAbsPath(name, path).exists())
4231                                 fix_relative_filename(name);
4232                         else
4233                                 cerr << "Warning: Could not find file '"
4234                                      << name << "'." << endl;
4235                         // write output
4236                         context.check_layout(os);
4237                         begin_inset(os, "External\n\ttemplate ");
4238                         os << "PDFPages\n\tfilename "
4239                            << name << "\n";
4240                         // parse the options
4241                         if (opts.find("pages") != opts.end())
4242                                 os << "\textra LaTeX \"pages="
4243                                    << opts["pages"] << "\"\n";
4244                         if (opts.find("angle") != opts.end())
4245                                 os << "\trotateAngle "
4246                                    << opts["angle"] << '\n';
4247                         if (opts.find("origin") != opts.end()) {
4248                                 ostringstream ss;
4249                                 string const opt = opts["origin"];
4250                                 if (opt == "tl") ss << "topleft";
4251                                 if (opt == "bl") ss << "bottomleft";
4252                                 if (opt == "Bl") ss << "baselineleft";
4253                                 if (opt == "c") ss << "center";
4254                                 if (opt == "tc") ss << "topcenter";
4255                                 if (opt == "bc") ss << "bottomcenter";
4256                                 if (opt == "Bc") ss << "baselinecenter";
4257                                 if (opt == "tr") ss << "topright";
4258                                 if (opt == "br") ss << "bottomright";
4259                                 if (opt == "Br") ss << "baselineright";
4260                                 if (!ss.str().empty())
4261                                         os << "\trotateOrigin " << ss.str() << '\n';
4262                                 else
4263                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
4264                         }
4265                         if (opts.find("width") != opts.end())
4266                                 os << "\twidth "
4267                                    << translate_len(opts["width"]) << '\n';
4268                         if (opts.find("height") != opts.end())
4269                                 os << "\theight "
4270                                    << translate_len(opts["height"]) << '\n';
4271                         if (opts.find("keepaspectratio") != opts.end())
4272                                 os << "\tkeepAspectRatio\n";
4273                         end_inset(os);
4274                         context.check_layout(os);
4275                         registerExternalTemplatePackages("PDFPages");
4276                 }
4277
4278                 else if (t.cs() == "loadgame") {
4279                         p.skip_spaces();
4280                         string name = normalize_filename(p.verbatim_item());
4281                         string const path = getMasterFilePath();
4282                         // We want to preserve relative / absolute filenames,
4283                         // therefore path is only used for testing
4284                         if (!makeAbsPath(name, path).exists()) {
4285                                 // The file extension is probably missing.
4286                                 // Now try to find it out.
4287                                 char const * const lyxskak_format[] = {"fen", 0};
4288                                 string const lyxskak_name =
4289                                         find_file(name, path, lyxskak_format);
4290                                 if (!lyxskak_name.empty())
4291                                         name = lyxskak_name;
4292                         }
4293                         if (makeAbsPath(name, path).exists())
4294                                 fix_relative_filename(name);
4295                         else
4296                                 cerr << "Warning: Could not find file '"
4297                                      << name << "'." << endl;
4298                         context.check_layout(os);
4299                         begin_inset(os, "External\n\ttemplate ");
4300                         os << "ChessDiagram\n\tfilename "
4301                            << name << "\n";
4302                         end_inset(os);
4303                         context.check_layout(os);
4304                         // after a \loadgame follows a \showboard
4305                         if (p.get_token().asInput() == "showboard")
4306                                 p.get_token();
4307                         registerExternalTemplatePackages("ChessDiagram");
4308                 }
4309
4310                 else {
4311                         // try to see whether the string is in unicodesymbols
4312                         // Only use text mode commands, since we are in text mode here,
4313                         // and math commands may be invalid (bug 6797)
4314                         bool termination;
4315                         docstring rem;
4316                         set<string> req;
4317                         docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
4318                                         Encodings::TEXT_CMD, termination, rem, &req);
4319                         if (!s.empty()) {
4320                                 if (!rem.empty())
4321                                         cerr << "When parsing " << t.cs()
4322                                              << ", result is " << to_utf8(s)
4323                                              << "+" << to_utf8(rem) << endl;
4324                                 context.check_layout(os);
4325                                 os << to_utf8(s);
4326                                 if (termination)
4327                                         skip_spaces_braces(p);
4328                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
4329                                         preamble.registerAutomaticallyLoadedPackage(*it);
4330                         }
4331                         //cerr << "#: " << t << " mode: " << mode << endl;
4332                         // heuristic: read up to next non-nested space
4333                         /*
4334                         string s = t.asInput();
4335                         string z = p.verbatim_item();
4336                         while (p.good() && z != " " && z.size()) {
4337                                 //cerr << "read: " << z << endl;
4338                                 s += z;
4339                                 z = p.verbatim_item();
4340                         }
4341                         cerr << "found ERT: " << s << endl;
4342                         handle_ert(os, s + ' ', context);
4343                         */
4344                         else {
4345                                 string name = t.asInput();
4346                                 if (p.next_token().asInput() == "*") {
4347                                         // Starred commands like \vspace*{}
4348                                         p.get_token();  // Eat '*'
4349                                         name += '*';
4350                                 }
4351                                 if (!parse_command(name, p, os, outer, context))
4352                                         handle_ert(os, name, context);
4353                         }
4354                 }
4355
4356                 if (flags & FLAG_LEAVE) {
4357                         flags &= ~FLAG_LEAVE;
4358                         break;
4359                 }
4360         }
4361 }
4362
4363 // }])
4364
4365
4366 } // namespace lyx