src/tex2lyx/text.cpp

   1 /**
   2  * \file tex2lyx/text.cpp
   3  * This file is part of LyX, the document processor.
   4  * Licence details can be found in the file COPYING.
   5  *
   6  * \author André Pönitz
   7  * \author Jean-Marc Lasgouttes
   8  * \author Uwe Stöhr
   9  *
  10  * Full author contact details are available in file CREDITS.
  11  */
  12
  13 // {[(
  14
  15 #include <config.h>
  16
  17 #include "tex2lyx.h"
  18
  19 #include "Context.h"
  20 #include "Encoding.h"
  21 #include "FloatList.h"
  22 #include "LaTeXPackages.h"
  23 #include "Layout.h"
  24 #include "Length.h"
  25 #include "Preamble.h"
  26
  27 #include "insets/ExternalTemplate.h"
  28
  29 #include "support/lassert.h"
  30 #include "support/convert.h"
  31 #include "support/FileName.h"
  32 #include "support/filetools.h"
  33 #include "support/lstrings.h"
  34 #include "support/lyxtime.h"
  35
  36 #include <algorithm>
  37 #include <iostream>
  38 #include <map>
  39 #include <sstream>
  40 #include <vector>
  41
  42 using namespace std;
  43 using namespace lyx::support;
  44
  45 namespace lyx {
  46
  47
  48 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  49                 Context const & context, InsetLayout const * layout)
  50 {
  51         bool const forcePlainLayout =
  52                 layout ? layout->forcePlainLayout() : false;
  53         Context newcontext(true, context.textclass);
  54         if (forcePlainLayout)
  55                 newcontext.layout = &context.textclass.plainLayout();
  56         else
  57                 newcontext.font = context.font;
  58         parse_text(p, os, flags, outer, newcontext);
  59         newcontext.check_end_layout(os);
  60 }
  61
  62
  63 namespace {
  64
  65 void parse_text_in_inset(Parser & p, ostream & os, unsigned flags, bool outer,
  66                 Context const & context, string const & name)
  67 {
  68         InsetLayout const * layout = 0;
  69         DocumentClass::InsetLayouts::const_iterator it =
  70                 context.textclass.insetLayouts().find(from_ascii(name));
  71         if (it != context.textclass.insetLayouts().end())
  72                 layout = &(it->second);
  73         parse_text_in_inset(p, os, flags, outer, context, layout);
  74 }
  75
  76 /// parses a paragraph snippet, useful for example for \\emph{...}
  77 void parse_text_snippet(Parser & p, ostream & os, unsigned flags, bool outer,
  78                 Context & context)
  79 {
  80         Context newcontext(context);
  81         // Don't inherit the paragraph-level extra stuff
  82         newcontext.par_extra_stuff.clear();
  83         parse_text(p, os, flags, outer, newcontext);
  84         // Make sure that we don't create invalid .lyx files
  85         context.need_layout = newcontext.need_layout;
  86         context.need_end_layout = newcontext.need_end_layout;
  87 }
  88
  89
  90 /*!
  91  * Thin wrapper around parse_text_snippet() using a string.
  92  *
  93  * We completely ignore \c context.need_layout and \c context.need_end_layout,
  94  * because our return value is not used directly (otherwise the stream version
  95  * of parse_text_snippet() could be used). That means that the caller needs
  96  * to do layout management manually.
  97  * This is intended to parse text that does not create any layout changes.
  98  */
  99 string parse_text_snippet(Parser & p, unsigned flags, const bool outer,
 100                   Context & context)
 101 {
 102         Context newcontext(context);
 103         newcontext.need_layout = false;
 104         newcontext.need_end_layout = false;
 105         newcontext.new_layout_allowed = false;
 106         // Avoid warning by Context::~Context()
 107         newcontext.par_extra_stuff.clear();
 108         ostringstream os;
 109         parse_text_snippet(p, os, flags, outer, newcontext);
 110         return os.str();
 111 }
 112
 113
 114 char const * const known_ref_commands[] = { "ref", "pageref", "vref",
 115  "vpageref", "prettyref", "eqref", 0 };
 116
 117 char const * const known_coded_ref_commands[] = { "ref", "pageref", "vref",
 118  "vpageref", "formatted", "eqref", 0 };
 119
 120 /**
 121  * known polyglossia language names (including variants)
 122  */
 123 const char * const polyglossia_languages[] = {
 124 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 125 "nynorsk", "syriac", "arabic", "danish", "icelandic", "occitan", "tamil",
 126 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 127 "irish", "portuges", "thai", "bahasai", "english", "italian", "romanian", "turkish",
 128 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 129 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazil",
 130 "brazilian", "finnish", "lithuanian", "scottish", "usorbian", "breton", "french",
 131 "lsorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 132 "welsh", "catalan", "german", "malayalam", "slovenian", "coptic", "greek",
 133 "marathi", "spanish",
 134 "american", "ancient", "australian", "british", "monotonic", "newzealand",
 135 "polytonic", 0};
 136
 137 /**
 138  * the same as polyglossia_languages with .lyx names
 139  * please keep this in sync with polyglossia_languages line by line!
 140  */
 141 const char * const coded_polyglossia_languages[] = {
 142 "albanian", "croatian", "hebrew", "norsk", "swedish", "amharic", "czech", "hindi",
 143 "nynorsk", "syriac", "arabic_arabi", "danish", "icelandic", "occitan", "tamil",
 144 "armenian", "divehi", "interlingua", "polish", "telugu", "asturian", "dutch",
 145 "irish", "portuges", "thai", "bahasa", "english", "italian", "romanian", "turkish",
 146 "bahasam", "esperanto", "lao", "russian", "turkmen", "basque", "estonian", "latin",
 147 "samin", "ukrainian", "bengali", "farsi", "latvian", "sanskrit", "urdu", "brazilian",
 148 "brazilian", "finnish", "lithuanian", "scottish", "uppersorbian", "breton", "french",
 149 "lowersorbian", "serbian", "vietnamese", "bulgarian", "galician", "magyar", "slovak",
 150 "welsh", "catalan", "ngerman", "malayalam", "slovene", "coptic", "greek",
 151 "marathi", "spanish",
 152 "american", "ancientgreek", "australian", "british", "greek", "newzealand",
 153 "polutonikogreek", 0};
 154
 155 /**
 156  * supported CJK encodings
 157  */
 158 const char * const supported_CJK_encodings[] = {
 159 "EUC-JP", "KS", "GB", "UTF8", 0};
 160
 161 /**
 162  * the same as supported_CJK_encodings with their corresponding LyX language name
 163  * please keep this in sync with supported_CJK_encodings line by line!
 164  */
 165 const char * const coded_supported_CJK_encodings[] = {
 166 "japanese-cjk", "korean", "chinese-simplified", "chinese-traditional", 0};
 167
 168 string CJK2lyx(string const & encoding)
 169 {
 170         char const * const * where = is_known(encoding, supported_CJK_encodings);
 171         if (where)
 172                 return coded_supported_CJK_encodings[where - supported_CJK_encodings];
 173         return encoding;
 174 }
 175
 176 /*!
 177  * natbib commands.
 178  * The starred forms are also known except for "citefullauthor",
 179  * "citeyear" and "citeyearpar".
 180  */
 181 char const * const known_natbib_commands[] = { "cite", "citet", "citep",
 182 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 183 "citefullauthor", "Citet", "Citep", "Citealt", "Citealp", "Citeauthor", 0 };
 184
 185 /*!
 186  * jurabib commands.
 187  * No starred form other than "cite*" known.
 188  */
 189 char const * const known_jurabib_commands[] = { "cite", "citet", "citep",
 190 "citealt", "citealp", "citeauthor", "citeyear", "citeyearpar",
 191 // jurabib commands not (yet) supported by LyX:
 192 // "fullcite",
 193 // "footcite", "footcitet", "footcitep", "footcitealt", "footcitealp",
 194 // "footciteauthor", "footciteyear", "footciteyearpar",
 195 "citefield", "citetitle", 0 };
 196
 197 /// LaTeX names for quotes
 198 char const * const known_quotes[] = { "dq", "guillemotleft", "flqq", "og",
 199 "guillemotright", "frqq", "fg", "glq", "glqq", "textquoteleft", "grq", "grqq",
 200 "quotedblbase", "textquotedblleft", "quotesinglbase", "textquoteright", "flq",
 201 "guilsinglleft", "frq", "guilsinglright", 0};
 202
 203 /// the same as known_quotes with .lyx names
 204 char const * const known_coded_quotes[] = { "prd", "ard", "ard", "ard",
 205 "ald", "ald", "ald", "gls", "gld", "els", "els", "grd",
 206 "gld", "grd", "gls", "ers", "fls",
 207 "fls", "frs", "frs", 0};
 208
 209 /// LaTeX names for font sizes
 210 char const * const known_sizes[] = { "tiny", "scriptsize", "footnotesize",
 211 "small", "normalsize", "large", "Large", "LARGE", "huge", "Huge", 0};
 212
 213 /// the same as known_sizes with .lyx names
 214 char const * const known_coded_sizes[] = { "tiny", "scriptsize", "footnotesize",
 215 "small", "normal", "large", "larger", "largest", "huge", "giant", 0};
 216
 217 /// LaTeX 2.09 names for font families
 218 char const * const known_old_font_families[] = { "rm", "sf", "tt", 0};
 219
 220 /// LaTeX names for font families
 221 char const * const known_font_families[] = { "rmfamily", "sffamily",
 222 "ttfamily", 0};
 223
 224 /// LaTeX names for font family changing commands
 225 char const * const known_text_font_families[] = { "textrm", "textsf",
 226 "texttt", 0};
 227
 228 /// The same as known_old_font_families, known_font_families and
 229 /// known_text_font_families with .lyx names
 230 char const * const known_coded_font_families[] = { "roman", "sans",
 231 "typewriter", 0};
 232
 233 /// LaTeX 2.09 names for font series
 234 char const * const known_old_font_series[] = { "bf", 0};
 235
 236 /// LaTeX names for font series
 237 char const * const known_font_series[] = { "bfseries", "mdseries", 0};
 238
 239 /// LaTeX names for font series changing commands
 240 char const * const known_text_font_series[] = { "textbf", "textmd", 0};
 241
 242 /// The same as known_old_font_series, known_font_series and
 243 /// known_text_font_series with .lyx names
 244 char const * const known_coded_font_series[] = { "bold", "medium", 0};
 245
 246 /// LaTeX 2.09 names for font shapes
 247 char const * const known_old_font_shapes[] = { "it", "sl", "sc", 0};
 248
 249 /// LaTeX names for font shapes
 250 char const * const known_font_shapes[] = { "itshape", "slshape", "scshape",
 251 "upshape", 0};
 252
 253 /// LaTeX names for font shape changing commands
 254 char const * const known_text_font_shapes[] = { "textit", "textsl", "textsc",
 255 "textup", 0};
 256
 257 /// The same as known_old_font_shapes, known_font_shapes and
 258 /// known_text_font_shapes with .lyx names
 259 char const * const known_coded_font_shapes[] = { "italic", "slanted",
 260 "smallcaps", "up", 0};
 261
 262 /// Known special characters which need skip_spaces_braces() afterwards
 263 char const * const known_special_chars[] = {"ldots", "lyxarrow",
 264 "textcompwordmark", "slash", 0};
 265
 266 /// the same as known_special_chars with .lyx names
 267 char const * const known_coded_special_chars[] = {"ldots{}", "menuseparator",
 268 "textcompwordmark{}", "slash{}", 0};
 269
 270 /*!
 271  * Graphics file extensions known by the dvips driver of the graphics package.
 272  * These extensions are used to complete the filename of an included
 273  * graphics file if it does not contain an extension.
 274  * The order must be the same that latex uses to find a file, because we
 275  * will use the first extension that matches.
 276  * This is only an approximation for the common cases. If we would want to
 277  * do it right in all cases, we would need to know which graphics driver is
 278  * used and know the extensions of every driver of the graphics package.
 279  */
 280 char const * const known_dvips_graphics_formats[] = {"eps", "ps", "eps.gz",
 281 "ps.gz", "eps.Z", "ps.Z", 0};
 282
 283 /*!
 284  * Graphics file extensions known by the pdftex driver of the graphics package.
 285  * \sa known_dvips_graphics_formats
 286  */
 287 char const * const known_pdftex_graphics_formats[] = {"png", "pdf", "jpg",
 288 "mps", "tif", 0};
 289
 290 /*!
 291  * Known file extensions for TeX files as used by \\include.
 292  */
 293 char const * const known_tex_extensions[] = {"tex", 0};
 294
 295 /// spaces known by InsetSpace
 296 char const * const known_spaces[] = { " ", "space", ",",
 297 "thinspace", "quad", "qquad", "enspace", "enskip",
 298 "negthinspace", "negmedspace", "negthickspace", "textvisiblespace",
 299 "hfill", "dotfill", "hrulefill", "leftarrowfill", "rightarrowfill",
 300 "upbracefill", "downbracefill", 0};
 301
 302 /// the same as known_spaces with .lyx names
 303 char const * const known_coded_spaces[] = { "space{}", "space{}",
 304 "thinspace{}", "thinspace{}", "quad{}", "qquad{}", "enspace{}", "enskip{}",
 305 "negthinspace{}", "negmedspace{}", "negthickspace{}", "textvisiblespace{}",
 306 "hfill{}", "dotfill{}", "hrulefill{}", "leftarrowfill{}", "rightarrowfill{}",
 307 "upbracefill{}", "downbracefill{}", 0};
 308
 309 /// These are translated by LyX to commands like "\\LyX{}", so we have to put
 310 /// them in ERT. "LaTeXe" must come before "LaTeX"!
 311 char const * const known_phrases[] = {"LyX", "TeX", "LaTeXe", "LaTeX", 0};
 312 char const * const known_coded_phrases[] = {"LyX", "TeX", "LaTeX2e", "LaTeX", 0};
 313 int const known_phrase_lengths[] = {3, 5, 7, 0};
 314
 315 // string to store the float type to be able to determine the type of subfloats
 316 string float_type = "";
 317
 318
 319 /// splits "x=z, y=b" into a map and an ordered keyword vector
 320 void split_map(string const & s, map<string, string> & res, vector<string> & keys)
 321 {
 322         vector<string> v;
 323         split(s, v);
 324         res.clear();
 325         keys.resize(v.size());
 326         for (size_t i = 0; i < v.size(); ++i) {
 327                 size_t const pos   = v[i].find('=');
 328                 string const index = trimSpaceAndEol(v[i].substr(0, pos));
 329                 string const value = trimSpaceAndEol(v[i].substr(pos + 1, string::npos));
 330                 res[index] = value;
 331                 keys[i] = index;
 332         }
 333 }
 334
 335
 336 /*!
 337  * Split a LaTeX length into value and unit.
 338  * The latter can be a real unit like "pt", or a latex length variable
 339  * like "\textwidth". The unit may contain additional stuff like glue
 340  * lengths, but we don't care, because such lengths are ERT anyway.
 341  * \returns true if \p value and \p unit are valid.
 342  */
 343 bool splitLatexLength(string const & len, string & value, string & unit)
 344 {
 345         if (len.empty())
 346                 return false;
 347         const string::size_type i = len.find_first_not_of(" -+0123456789.,");
 348         //'4,5' is a valid LaTeX length number. Change it to '4.5'
 349         string const length = subst(len, ',', '.');
 350         if (i == string::npos)
 351                 return false;
 352         if (i == 0) {
 353                 if (len[0] == '\\') {
 354                         // We had something like \textwidth without a factor
 355                         value = "1.0";
 356                 } else {
 357                         return false;
 358                 }
 359         } else {
 360                 value = trimSpaceAndEol(string(length, 0, i));
 361         }
 362         if (value == "-")
 363                 value = "-1.0";
 364         // 'cM' is a valid LaTeX length unit. Change it to 'cm'
 365         if (contains(len, '\\'))
 366                 unit = trimSpaceAndEol(string(len, i));
 367         else
 368                 unit = ascii_lowercase(trimSpaceAndEol(string(len, i)));
 369         return true;
 370 }
 371
 372
 373 /// A simple function to translate a latex length to something LyX can
 374 /// understand. Not perfect, but rather best-effort.
 375 bool translate_len(string const & length, string & valstring, string & unit)
 376 {
 377         if (!splitLatexLength(length, valstring, unit))
 378                 return false;
 379         // LyX uses percent values
 380         double value;
 381         istringstream iss(valstring);
 382         iss >> value;
 383         value *= 100;
 384         ostringstream oss;
 385         oss << value;
 386         string const percentval = oss.str();
 387         // a normal length
 388         if (unit.empty() || unit[0] != '\\')
 389                 return true;
 390         string::size_type const i = unit.find(' ');
 391         string const endlen = (i == string::npos) ? string() : string(unit, i);
 392         if (unit == "\\textwidth") {
 393                 valstring = percentval;
 394                 unit = "text%" + endlen;
 395         } else if (unit == "\\columnwidth") {
 396                 valstring = percentval;
 397                 unit = "col%" + endlen;
 398         } else if (unit == "\\paperwidth") {
 399                 valstring = percentval;
 400                 unit = "page%" + endlen;
 401         } else if (unit == "\\linewidth") {
 402                 valstring = percentval;
 403                 unit = "line%" + endlen;
 404         } else if (unit == "\\paperheight") {
 405                 valstring = percentval;
 406                 unit = "pheight%" + endlen;
 407         } else if (unit == "\\textheight") {
 408                 valstring = percentval;
 409                 unit = "theight%" + endlen;
 410         }
 411         return true;
 412 }
 413
 414 }
 415
 416
 417 string translate_len(string const & length)
 418 {
 419         string unit;
 420         string value;
 421         if (translate_len(length, value, unit))
 422                 return value + unit;
 423         // If the input is invalid, return what we have.
 424         return length;
 425 }
 426
 427
 428 namespace {
 429
 430 /*!
 431  * Translates a LaTeX length into \p value, \p unit and
 432  * \p special parts suitable for a box inset.
 433  * The difference from translate_len() is that a box inset knows about
 434  * some special "units" that are stored in \p special.
 435  */
 436 void translate_box_len(string const & length, string & value, string & unit, string & special)
 437 {
 438         if (translate_len(length, value, unit)) {
 439                 if (unit == "\\height" || unit == "\\depth" ||
 440                     unit == "\\totalheight" || unit == "\\width") {
 441                         special = unit.substr(1);
 442                         // The unit is not used, but LyX requires a dummy setting
 443                         unit = "in";
 444                 } else
 445                         special = "none";
 446         } else {
 447                 value.clear();
 448                 unit = length;
 449                 special = "none";
 450         }
 451 }
 452
 453
 454 /*!
 455  * Find a file with basename \p name in path \p path and an extension
 456  * in \p extensions.
 457  */
 458 string find_file(string const & name, string const & path,
 459                  char const * const * extensions)
 460 {
 461         for (char const * const * what = extensions; *what; ++what) {
 462                 string const trial = addExtension(name, *what);
 463                 if (makeAbsPath(trial, path).exists())
 464                         return trial;
 465         }
 466         return string();
 467 }
 468
 469
 470 void begin_inset(ostream & os, string const & name)
 471 {
 472         os << "\n\\begin_inset " << name;
 473 }
 474
 475
 476 void begin_command_inset(ostream & os, string const & name,
 477                          string const & latexname)
 478 {
 479         begin_inset(os, "CommandInset ");
 480         os << name << "\nLatexCommand " << latexname << '\n';
 481 }
 482
 483
 484 void end_inset(ostream & os)
 485 {
 486         os << "\n\\end_inset\n\n";
 487 }
 488
 489
 490 bool skip_braces(Parser & p)
 491 {
 492         if (p.next_token().cat() != catBegin)
 493                 return false;
 494         p.get_token();
 495         if (p.next_token().cat() == catEnd) {
 496                 p.get_token();
 497                 return true;
 498         }
 499         p.putback();
 500         return false;
 501 }
 502
 503
 504 /// replace LaTeX commands in \p s from the unicodesymbols file with their
 505 /// unicode points
 506 docstring convert_unicodesymbols(docstring s)
 507 {
 508         odocstringstream os;
 509         for (size_t i = 0; i < s.size();) {
 510                 if (s[i] != '\\') {
 511                         os.put(s[i++]);
 512                         continue;
 513                 }
 514                 s = s.substr(i);
 515                 bool termination;
 516                 docstring rem;
 517                 set<string> req;
 518                 docstring parsed = encodings.fromLaTeXCommand(s,
 519                                 Encodings::TEXT_CMD, termination, rem, &req);
 520                 set<string>::const_iterator it = req.begin();
 521                 set<string>::const_iterator en = req.end();
 522                 for (; it != en; ++it)
 523                         preamble.registerAutomaticallyLoadedPackage(*it);
 524                 os << parsed;
 525                 s = rem;
 526                 if (s.empty() || s[0] != '\\')
 527                         i = 0;
 528                 else
 529                         i = 1;
 530         }
 531         return os.str();
 532 }
 533
 534
 535 /// try to convert \p s to a valid InsetCommand argument
 536 string convert_command_inset_arg(string s)
 537 {
 538         if (isAscii(s))
 539                 // since we don't know the input encoding we can't use from_utf8
 540                 s = to_utf8(convert_unicodesymbols(from_ascii(s)));
 541         // LyX cannot handle newlines in a latex command
 542         return subst(s, "\n", " ");
 543 }
 544
 545
 546 void handle_backslash(ostream & os, string const & s)
 547 {
 548         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 549                 if (*it == '\\')
 550                         os << "\n\\backslash\n";
 551                 else
 552                         os << *it;
 553         }
 554 }
 555
 556
 557 void handle_ert(ostream & os, string const & s, Context & context)
 558 {
 559         // We must have a valid layout before outputting the ERT inset.
 560         context.check_layout(os);
 561         Context newcontext(true, context.textclass);
 562         begin_inset(os, "ERT");
 563         os << "\nstatus collapsed\n";
 564         newcontext.check_layout(os);
 565         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
 566                 if (*it == '\\')
 567                         os << "\n\\backslash\n";
 568                 else if (*it == '\n') {
 569                         newcontext.new_paragraph(os);
 570                         newcontext.check_layout(os);
 571                 } else
 572                         os << *it;
 573         }
 574         newcontext.check_end_layout(os);
 575         end_inset(os);
 576 }
 577
 578
 579 void handle_comment(ostream & os, string const & s, Context & context)
 580 {
 581         // TODO: Handle this better
 582         Context newcontext(true, context.textclass);
 583         begin_inset(os, "ERT");
 584         os << "\nstatus collapsed\n";
 585         newcontext.check_layout(os);
 586         handle_backslash(os, s);
 587         // make sure that our comment is the last thing on the line
 588         newcontext.new_paragraph(os);
 589         newcontext.check_layout(os);
 590         newcontext.check_end_layout(os);
 591         end_inset(os);
 592 }
 593
 594
 595 Layout const * findLayout(TextClass const & textclass, string const & name, bool command)
 596 {
 597         Layout const * layout = findLayoutWithoutModule(textclass, name, command);
 598         if (layout)
 599                 return layout;
 600         if (checkModule(name, command))
 601                 return findLayoutWithoutModule(textclass, name, command);
 602         return layout;
 603 }
 604
 605
 606 InsetLayout const * findInsetLayout(TextClass const & textclass, string const & name, bool command)
 607 {
 608         InsetLayout const * insetlayout = findInsetLayoutWithoutModule(textclass, name, command);
 609         if (insetlayout)
 610                 return insetlayout;
 611         if (checkModule(name, command))
 612                 return findInsetLayoutWithoutModule(textclass, name, command);
 613         return insetlayout;
 614 }
 615
 616
 617 void eat_whitespace(Parser &, ostream &, Context &, bool);
 618
 619
 620 /*!
 621  * Skips whitespace and braces.
 622  * This should be called after a command has been parsed that is not put into
 623  * ERT, and where LyX adds "{}" if needed.
 624  */
 625 void skip_spaces_braces(Parser & p, bool keepws = false)
 626 {
 627         /* The following four examples produce the same typeset output and
 628            should be handled by this function:
 629            - abc \j{} xyz
 630            - abc \j {} xyz
 631            - abc \j
 632              {} xyz
 633            - abc \j %comment
 634              {} xyz
 635          */
 636         // Unfortunately we need to skip comments, too.
 637         // We can't use eat_whitespace since writing them after the {}
 638         // results in different output in some cases.
 639         bool const skipped_spaces = p.skip_spaces(true);
 640         bool const skipped_braces = skip_braces(p);
 641         if (keepws && skipped_spaces && !skipped_braces)
 642                 // put back the space (it is better handled by check_space)
 643                 p.unskip_spaces(true);
 644 }
 645
 646
 647 void output_command_layout(ostream & os, Parser & p, bool outer,
 648                            Context & parent_context,
 649                            Layout const * newlayout)
 650 {
 651         TeXFont const oldFont = parent_context.font;
 652         // save the current font size
 653         string const size = oldFont.size;
 654         // reset the font size to default, because the font size switches
 655         // don't affect section headings and the like
 656         parent_context.font.size = Context::normalfont.size;
 657         // we only need to write the font change if we have an open layout
 658         if (!parent_context.atParagraphStart())
 659                 output_font_change(os, oldFont, parent_context.font);
 660         parent_context.check_end_layout(os);
 661         Context context(true, parent_context.textclass, newlayout,
 662                         parent_context.layout, parent_context.font);
 663         if (parent_context.deeper_paragraph) {
 664                 // We are beginning a nested environment after a
 665                 // deeper paragraph inside the outer list environment.
 666                 // Therefore we don't need to output a "begin deeper".
 667                 context.need_end_deeper = true;
 668         }
 669         context.check_deeper(os);
 670         context.check_layout(os);
 671         unsigned int optargs = 0;
 672         while (optargs < context.layout->optargs) {
 673                 eat_whitespace(p, os, context, false);
 674                 if (p.next_token().cat() == catEscape ||
 675                     p.next_token().character() != '[')
 676                         break;
 677                 p.get_token(); // eat '['
 678                 begin_inset(os, "Argument\n");
 679                 os << "status collapsed\n\n";
 680                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
 681                 end_inset(os);
 682                 eat_whitespace(p, os, context, false);
 683                 ++optargs;
 684         }
 685         unsigned int reqargs = 0;
 686         while (reqargs < context.layout->reqargs) {
 687                 eat_whitespace(p, os, context, false);
 688                 if (p.next_token().cat() != catBegin)
 689                         break;
 690                 p.get_token(); // eat '{'
 691                 begin_inset(os, "Argument\n");
 692                 os << "status collapsed\n\n";
 693                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
 694                 end_inset(os);
 695                 eat_whitespace(p, os, context, false);
 696                 ++reqargs;
 697         }
 698         parse_text(p, os, FLAG_ITEM, outer, context);
 699         context.check_end_layout(os);
 700         if (parent_context.deeper_paragraph) {
 701                 // We must suppress the "end deeper" because we
 702                 // suppressed the "begin deeper" above.
 703                 context.need_end_deeper = false;
 704         }
 705         context.check_end_deeper(os);
 706         // We don't need really a new paragraph, but
 707         // we must make sure that the next item gets a \begin_layout.
 708         parent_context.new_paragraph(os);
 709         // Set the font size to the original value. No need to output it here
 710         // (Context::begin_layout() will do that if needed)
 711         parent_context.font.size = size;
 712 }
 713
 714
 715 /*!
 716  * Output a space if necessary.
 717  * This function gets called for every whitespace token.
 718  *
 719  * We have three cases here:
 720  * 1. A space must be suppressed. Example: The lyxcode case below
 721  * 2. A space may be suppressed. Example: Spaces before "\par"
 722  * 3. A space must not be suppressed. Example: A space between two words
 723  *
 724  * We currently handle only 1. and 3 and from 2. only the case of
 725  * spaces before newlines as a side effect.
 726  *
 727  * 2. could be used to suppress as many spaces as possible. This has two effects:
 728  * - Reimporting LyX generated LaTeX files changes almost no whitespace
 729  * - Superflous whitespace from non LyX generated LaTeX files is removed.
 730  * The drawback is that the logic inside the function becomes
 731  * complicated, and that is the reason why it is not implemented.
 732  */
 733 void check_space(Parser & p, ostream & os, Context & context)
 734 {
 735         Token const next = p.next_token();
 736         Token const curr = p.curr_token();
 737         // A space before a single newline and vice versa must be ignored
 738         // LyX emits a newline before \end{lyxcode}.
 739         // This newline must be ignored,
 740         // otherwise LyX will add an additional protected space.
 741         if (next.cat() == catSpace ||
 742             next.cat() == catNewline ||
 743             (next.cs() == "end" && context.layout->free_spacing && curr.cat() == catNewline)) {
 744                 return;
 745         }
 746         context.check_layout(os);
 747         os << ' ';
 748 }
 749
 750
 751 /*!
 752  * Parse all arguments of \p command
 753  */
 754 void parse_arguments(string const & command,
 755                      vector<ArgumentType> const & template_arguments,
 756                      Parser & p, ostream & os, bool outer, Context & context)
 757 {
 758         string ert = command;
 759         size_t no_arguments = template_arguments.size();
 760         for (size_t i = 0; i < no_arguments; ++i) {
 761                 switch (template_arguments[i]) {
 762                 case required:
 763                 case req_group:
 764                         // This argument contains regular LaTeX
 765                         handle_ert(os, ert + '{', context);
 766                         eat_whitespace(p, os, context, false);
 767                         if (template_arguments[i] == required)
 768                                 parse_text(p, os, FLAG_ITEM, outer, context);
 769                         else
 770                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
 771                         ert = "}";
 772                         break;
 773                 case item:
 774                         // This argument consists only of a single item.
 775                         // The presence of '{' or not must be preserved.
 776                         p.skip_spaces();
 777                         if (p.next_token().cat() == catBegin)
 778                                 ert += '{' + p.verbatim_item() + '}';
 779                         else
 780                                 ert += p.verbatim_item();
 781                         break;
 782                 case displaymath:
 783                 case verbatim:
 784                         // This argument may contain special characters
 785                         ert += '{' + p.verbatim_item() + '}';
 786                         break;
 787                 case optional:
 788                 case opt_group:
 789                         // true because we must not eat whitespace
 790                         // if an optional arg follows we must not strip the
 791                         // brackets from this one
 792                         if (i < no_arguments - 1 &&
 793                             template_arguments[i+1] == optional)
 794                                 ert += p.getFullOpt(true);
 795                         else
 796                                 ert += p.getOpt(true);
 797                         break;
 798                 }
 799         }
 800         handle_ert(os, ert, context);
 801 }
 802
 803
 804 /*!
 805  * Check whether \p command is a known command. If yes,
 806  * handle the command with all arguments.
 807  * \return true if the command was parsed, false otherwise.
 808  */
 809 bool parse_command(string const & command, Parser & p, ostream & os,
 810                    bool outer, Context & context)
 811 {
 812         if (known_commands.find(command) != known_commands.end()) {
 813                 parse_arguments(command, known_commands[command], p, os,
 814                                 outer, context);
 815                 return true;
 816         }
 817         return false;
 818 }
 819
 820
 821 /// Parses a minipage or parbox
 822 void parse_box(Parser & p, ostream & os, unsigned outer_flags,
 823                unsigned inner_flags, bool outer, Context & parent_context,
 824                string const & outer_type, string const & special,
 825                string const & inner_type)
 826 {
 827         string position;
 828         string inner_pos;
 829         string hor_pos = "c";
 830         // We need to set the height to the LaTeX default of 1\\totalheight
 831         // for the case when no height argument is given
 832         string height_value = "1";
 833         string height_unit = "in";
 834         string height_special = "totalheight";
 835         string latex_height;
 836         string width_value;
 837         string width_unit;
 838         string latex_width;
 839         string width_special = "none";
 840         if (!inner_type.empty() && p.hasOpt()) {
 841                 if (inner_type != "makebox")
 842                         position = p.getArg('[', ']');
 843                 else {
 844                         latex_width = p.getArg('[', ']');
 845                         translate_box_len(latex_width, width_value, width_unit, width_special);
 846                         position = "t";
 847                 }
 848                 if (position != "t" && position != "c" && position != "b") {
 849                         cerr << "invalid position " << position << " for "
 850                              << inner_type << endl;
 851                         position = "c";
 852                 }
 853                 if (p.hasOpt()) {
 854                         if (inner_type != "makebox") {
 855                                 latex_height = p.getArg('[', ']');
 856                                 translate_box_len(latex_height, height_value, height_unit, height_special);
 857                         } else
 858                                 hor_pos = p.getArg('[', ']');
 859
 860                         if (p.hasOpt()) {
 861                                 inner_pos = p.getArg('[', ']');
 862                                 if (inner_pos != "c" && inner_pos != "t" &&
 863                                     inner_pos != "b" && inner_pos != "s") {
 864                                         cerr << "invalid inner_pos "
 865                                              << inner_pos << " for "
 866                                              << inner_type << endl;
 867                                         inner_pos = position;
 868                                 }
 869                         }
 870                 }
 871         }
 872         if (inner_type.empty()) {
 873                 if (special.empty() && outer_type != "framebox")
 874                         latex_width = "1\\columnwidth";
 875                 else {
 876                         Parser p2(special);
 877                         latex_width = p2.getArg('[', ']');
 878                         string const opt = p2.getArg('[', ']');
 879                         if (!opt.empty()) {
 880                                 hor_pos = opt;
 881                                 if (hor_pos != "l" && hor_pos != "c" &&
 882                                     hor_pos != "r") {
 883                                         cerr << "invalid hor_pos " << hor_pos
 884                                              << " for " << outer_type << endl;
 885                                         hor_pos = "c";
 886                                 }
 887                         }
 888                 }
 889         } else if (inner_type != "makebox")
 890                 latex_width = p.verbatim_item();
 891         // if e.g. only \ovalbox{content} was used, set the width to 1\columnwidth
 892         // as this is LyX's standard for such cases (except for makebox)
 893         // \framebox is more special and handled below
 894         if (latex_width.empty() && inner_type != "makebox"
 895                 && outer_type != "framebox")
 896                 latex_width = "1\\columnwidth";
 897
 898         translate_len(latex_width, width_value, width_unit);
 899
 900         bool shadedparbox = false;
 901         if (inner_type == "shaded") {
 902                 eat_whitespace(p, os, parent_context, false);
 903                 if (outer_type == "parbox") {
 904                         // Eat '{'
 905                         if (p.next_token().cat() == catBegin)
 906                                 p.get_token();
 907                         eat_whitespace(p, os, parent_context, false);
 908                         shadedparbox = true;
 909                 }
 910                 p.get_token();
 911                 p.getArg('{', '}');
 912         }
 913         // If we already read the inner box we have to push the inner env
 914         if (!outer_type.empty() && !inner_type.empty() &&
 915             (inner_flags & FLAG_END))
 916                 active_environments.push_back(inner_type);
 917         // LyX can't handle length variables
 918         bool use_ert = contains(width_unit, '\\') || contains(height_unit, '\\');
 919         if (!use_ert && !outer_type.empty() && !inner_type.empty()) {
 920                 // Look whether there is some content after the end of the
 921                 // inner box, but before the end of the outer box.
 922                 // If yes, we need to output ERT.
 923                 p.pushPosition();
 924                 if (inner_flags & FLAG_END)
 925                         p.verbatimEnvironment(inner_type);
 926                 else
 927                         p.verbatim_item();
 928                 p.skip_spaces(true);
 929                 bool const outer_env(outer_type == "framed" || outer_type == "minipage");
 930                 if ((outer_env && p.next_token().asInput() != "\\end") ||
 931                     (!outer_env && p.next_token().cat() != catEnd)) {
 932                         // something is between the end of the inner box and
 933                         // the end of the outer box, so we need to use ERT.
 934                         use_ert = true;
 935                 }
 936                 p.popPosition();
 937         }
 938         // if only \makebox{content} was used we can set its width to 1\width
 939         // because this identic and also identic to \mbox
 940         // this doesn't work for \framebox{content}, thus we have to use ERT for this
 941         if (latex_width.empty() && inner_type == "makebox") {
 942                 width_value = "1";
 943                 width_unit = "in";
 944                 width_special = "width";
 945         } else if (latex_width.empty() && outer_type == "framebox") {
 946                 use_ert = true;
 947         }
 948         if (use_ert) {
 949                 ostringstream ss;
 950                 if (!outer_type.empty()) {
 951                         if (outer_flags & FLAG_END)
 952                                 ss << "\\begin{" << outer_type << '}';
 953                         else {
 954                                 ss << '\\' << outer_type << '{';
 955                                 if (!special.empty())
 956                                         ss << special;
 957                         }
 958                 }
 959                 if (!inner_type.empty()) {
 960                         if (inner_type != "shaded") {
 961                                 if (inner_flags & FLAG_END)
 962                                         ss << "\\begin{" << inner_type << '}';
 963                                 else
 964                                         ss << '\\' << inner_type;
 965                         }
 966                         if (!position.empty())
 967                                 ss << '[' << position << ']';
 968                         if (!latex_height.empty())
 969                                 ss << '[' << latex_height << ']';
 970                         if (!inner_pos.empty())
 971                                 ss << '[' << inner_pos << ']';
 972                         ss << '{' << latex_width << '}';
 973                         if (!(inner_flags & FLAG_END))
 974                                 ss << '{';
 975                 }
 976                 if (inner_type == "shaded")
 977                         ss << "\\begin{shaded}";
 978                 handle_ert(os, ss.str(), parent_context);
 979                 if (!inner_type.empty()) {
 980                         parse_text(p, os, inner_flags, outer, parent_context);
 981                         if (inner_flags & FLAG_END)
 982                                 handle_ert(os, "\\end{" + inner_type + '}',
 983                                            parent_context);
 984                         else
 985                                 handle_ert(os, "}", parent_context);
 986                 }
 987                 if (!outer_type.empty()) {
 988                         // If we already read the inner box we have to pop
 989                         // the inner env
 990                         if (!inner_type.empty() && (inner_flags & FLAG_END))
 991                                 active_environments.pop_back();
 992
 993                         // Ensure that the end of the outer box is parsed correctly:
 994                         // The opening brace has been eaten by parse_outer_box()
 995                         if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
 996                                 outer_flags &= ~FLAG_ITEM;
 997                                 outer_flags |= FLAG_BRACE_LAST;
 998                         }
 999                         parse_text(p, os, outer_flags, outer, parent_context);
1000                         if (outer_flags & FLAG_END)
1001                                 handle_ert(os, "\\end{" + outer_type + '}',
1002                                            parent_context);
1003                         else if (inner_type.empty() && outer_type == "framebox")
1004                                 // in this case it is already closed later
1005                                 ;
1006                         else
1007                                 handle_ert(os, "}", parent_context);
1008                 }
1009         } else {
1010                 // LyX does not like empty positions, so we have
1011                 // to set them to the LaTeX default values here.
1012                 if (position.empty())
1013                         position = "c";
1014                 if (inner_pos.empty())
1015                         inner_pos = position;
1016                 parent_context.check_layout(os);
1017                 begin_inset(os, "Box ");
1018                 if (outer_type == "framed")
1019                         os << "Framed\n";
1020                 else if (outer_type == "framebox")
1021                         os << "Boxed\n";
1022                 else if (outer_type == "shadowbox")
1023                         os << "Shadowbox\n";
1024                 else if ((outer_type == "shaded" && inner_type.empty()) ||
1025                              (outer_type == "minipage" && inner_type == "shaded") ||
1026                              (outer_type == "parbox" && inner_type == "shaded")) {
1027                         os << "Shaded\n";
1028                         preamble.registerAutomaticallyLoadedPackage("color");
1029                 } else if (outer_type == "doublebox")
1030                         os << "Doublebox\n";
1031                 else if (outer_type.empty())
1032                         os << "Frameless\n";
1033                 else
1034                         os << outer_type << '\n';
1035                 os << "position \"" << position << "\"\n";
1036                 os << "hor_pos \"" << hor_pos << "\"\n";
1037                 os << "has_inner_box " << !inner_type.empty() << "\n";
1038                 os << "inner_pos \"" << inner_pos << "\"\n";
1039                 os << "use_parbox " << (inner_type == "parbox" || shadedparbox)
1040                    << '\n';
1041                 os << "use_makebox " << (inner_type == "makebox") << '\n';
1042                 os << "width \"" << width_value << width_unit << "\"\n";
1043                 os << "special \"" << width_special << "\"\n";
1044                 os << "height \"" << height_value << height_unit << "\"\n";
1045                 os << "height_special \"" << height_special << "\"\n";
1046                 os << "status open\n\n";
1047
1048                 // Unfortunately we can't use parse_text_in_inset:
1049                 // InsetBox::forcePlainLayout() is hard coded and does not
1050                 // use the inset layout. Apart from that do we call parse_text
1051                 // up to two times, but need only one check_end_layout.
1052                 bool const forcePlainLayout =
1053                         (!inner_type.empty() || inner_type == "makebox") &&
1054                         outer_type != "shaded" && outer_type != "framed";
1055                 Context context(true, parent_context.textclass);
1056                 if (forcePlainLayout)
1057                         context.layout = &context.textclass.plainLayout();
1058                 else
1059                         context.font = parent_context.font;
1060
1061                 // If we have no inner box the contents will be read with the outer box
1062                 if (!inner_type.empty())
1063                         parse_text(p, os, inner_flags, outer, context);
1064
1065                 // Ensure that the end of the outer box is parsed correctly:
1066                 // The opening brace has been eaten by parse_outer_box()
1067                 if (!outer_type.empty() && (outer_flags & FLAG_ITEM)) {
1068                         outer_flags &= ~FLAG_ITEM;
1069                         outer_flags |= FLAG_BRACE_LAST;
1070                 }
1071
1072                 // Find end of outer box, output contents if inner_type is
1073                 // empty and output possible comments
1074                 if (!outer_type.empty()) {
1075                         // If we already read the inner box we have to pop
1076                         // the inner env
1077                         if (!inner_type.empty() && (inner_flags & FLAG_END))
1078                                 active_environments.pop_back();
1079                         // This does not output anything but comments if
1080                         // inner_type is not empty (see use_ert)
1081                         parse_text(p, os, outer_flags, outer, context);
1082                 }
1083
1084                 context.check_end_layout(os);
1085                 end_inset(os);
1086 #ifdef PRESERVE_LAYOUT
1087                 // LyX puts a % after the end of the minipage
1088                 if (p.next_token().cat() == catNewline && p.next_token().cs().size() > 1) {
1089                         // new paragraph
1090                         //handle_comment(os, "%dummy", parent_context);
1091                         p.get_token();
1092                         p.skip_spaces();
1093                         parent_context.new_paragraph(os);
1094                 }
1095                 else if (p.next_token().cat() == catSpace || p.next_token().cat() == catNewline) {
1096                         //handle_comment(os, "%dummy", parent_context);
1097                         p.get_token();
1098                         p.skip_spaces();
1099                         // We add a protected space if something real follows
1100                         if (p.good() && p.next_token().cat() != catComment) {
1101                                 begin_inset(os, "space ~\n");
1102                                 end_inset(os);
1103                         }
1104                 }
1105 #endif
1106         }
1107 }
1108
1109
1110 void parse_outer_box(Parser & p, ostream & os, unsigned flags, bool outer,
1111                      Context & parent_context, string const & outer_type,
1112                      string const & special)
1113 {
1114         eat_whitespace(p, os, parent_context, false);
1115         if (flags & FLAG_ITEM) {
1116                 // Eat '{'
1117                 if (p.next_token().cat() == catBegin)
1118                         p.get_token();
1119                 else
1120                         cerr << "Warning: Ignoring missing '{' after \\"
1121                              << outer_type << '.' << endl;
1122                 eat_whitespace(p, os, parent_context, false);
1123         }
1124         string inner;
1125         unsigned int inner_flags = 0;
1126         p.pushPosition();
1127         if (outer_type == "minipage" || outer_type == "parbox") {
1128                 p.skip_spaces(true);
1129                 while (p.hasOpt()) {
1130                         p.getArg('[', ']');
1131                         p.skip_spaces(true);
1132                 }
1133                 p.getArg('{', '}');
1134                 p.skip_spaces(true);
1135                 if (outer_type == "parbox") {
1136                         // Eat '{'
1137                         if (p.next_token().cat() == catBegin)
1138                                 p.get_token();
1139                         p.skip_spaces(true);
1140                 }
1141         }
1142         if (outer_type == "shaded") {
1143                 // These boxes never have an inner box
1144                 ;
1145         } else if (p.next_token().asInput() == "\\parbox") {
1146                 inner = p.get_token().cs();
1147                 inner_flags = FLAG_ITEM;
1148         } else if (p.next_token().asInput() == "\\begin") {
1149                 // Is this a minipage or shaded box?
1150                 p.pushPosition();
1151                 p.get_token();
1152                 inner = p.getArg('{', '}');
1153                 p.popPosition();
1154                 if (inner == "minipage" || inner == "shaded")
1155                         inner_flags = FLAG_END;
1156                 else
1157                         inner = "";
1158         }
1159         p.popPosition();
1160         if (inner_flags == FLAG_END) {
1161                 if (inner != "shaded")
1162                 {
1163                         p.get_token();
1164                         p.getArg('{', '}');
1165                         eat_whitespace(p, os, parent_context, false);
1166                 }
1167                 parse_box(p, os, flags, FLAG_END, outer, parent_context,
1168                           outer_type, special, inner);
1169         } else {
1170                 if (inner_flags == FLAG_ITEM) {
1171                         p.get_token();
1172                         eat_whitespace(p, os, parent_context, false);
1173                 }
1174                 parse_box(p, os, flags, inner_flags, outer, parent_context,
1175                           outer_type, special, inner);
1176         }
1177 }
1178
1179
1180 void parse_listings(Parser & p, ostream & os, Context & parent_context, bool in_line)
1181 {
1182         parent_context.check_layout(os);
1183         begin_inset(os, "listings\n");
1184         if (p.hasOpt()) {
1185                 string arg = p.verbatimOption();
1186                 os << "lstparams " << '"' << arg << '"' << '\n';
1187         }
1188         if (in_line)
1189                 os << "inline true\n";
1190         else
1191                 os << "inline false\n";
1192         os << "status collapsed\n";
1193         Context context(true, parent_context.textclass);
1194         context.layout = &parent_context.textclass.plainLayout();
1195         string s;
1196         if (in_line) {
1197                 s = p.plainCommand('!', '!', "lstinline");
1198                 context.new_paragraph(os);
1199                 context.check_layout(os);
1200         } else
1201                 s = p.plainEnvironment("lstlisting");
1202         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1203                 if (*it == '\\')
1204                         os << "\n\\backslash\n";
1205                 else if (*it == '\n') {
1206                         // avoid adding an empty paragraph at the end
1207                         if (it + 1 != et) {
1208                                 context.new_paragraph(os);
1209                                 context.check_layout(os);
1210                         }
1211                 } else
1212                         os << *it;
1213         }
1214         context.check_end_layout(os);
1215         end_inset(os);
1216 }
1217
1218
1219 /// parse an unknown environment
1220 void parse_unknown_environment(Parser & p, string const & name, ostream & os,
1221                                unsigned flags, bool outer,
1222                                Context & parent_context)
1223 {
1224         if (name == "tabbing")
1225                 // We need to remember that we have to handle '\=' specially
1226                 flags |= FLAG_TABBING;
1227
1228         // We need to translate font changes and paragraphs inside the
1229         // environment to ERT if we have a non standard font.
1230         // Otherwise things like
1231         // \large\begin{foo}\huge bar\end{foo}
1232         // will not work.
1233         bool const specialfont =
1234                 (parent_context.font != parent_context.normalfont);
1235         bool const new_layout_allowed = parent_context.new_layout_allowed;
1236         if (specialfont)
1237                 parent_context.new_layout_allowed = false;
1238         handle_ert(os, "\\begin{" + name + "}", parent_context);
1239         parse_text_snippet(p, os, flags, outer, parent_context);
1240         handle_ert(os, "\\end{" + name + "}", parent_context);
1241         if (specialfont)
1242                 parent_context.new_layout_allowed = new_layout_allowed;
1243 }
1244
1245
1246 void parse_environment(Parser & p, ostream & os, bool outer,
1247                        string & last_env, Context & parent_context)
1248 {
1249         Layout const * newlayout;
1250         InsetLayout const * newinsetlayout = 0;
1251         string const name = p.getArg('{', '}');
1252         const bool is_starred = suffixIs(name, '*');
1253         string const unstarred_name = rtrim(name, "*");
1254         active_environments.push_back(name);
1255
1256         if (is_math_env(name)) {
1257                 parent_context.check_layout(os);
1258                 begin_inset(os, "Formula ");
1259                 os << "\\begin{" << name << "}";
1260                 parse_math(p, os, FLAG_END, MATH_MODE);
1261                 os << "\\end{" << name << "}";
1262                 end_inset(os);
1263                 if (is_display_math_env(name)) {
1264                         // Prevent the conversion of a line break to a space
1265                         // (bug 7668). This does not change the output, but
1266                         // looks ugly in LyX.
1267                         eat_whitespace(p, os, parent_context, false);
1268                 }
1269         }
1270
1271         else if (is_known(name, polyglossia_languages)) {
1272                 // We must begin a new paragraph if not already done
1273                 if (! parent_context.atParagraphStart()) {
1274                         parent_context.check_end_layout(os);
1275                         parent_context.new_paragraph(os);
1276                 }
1277                 // save the language in the context so that it is
1278                 // handled by parse_text
1279                 parent_context.font.language = polyglossia2lyx(name);
1280                 parse_text(p, os, FLAG_END, outer, parent_context);
1281                 // Just in case the environment is empty
1282                 parent_context.extra_stuff.erase();
1283                 // We must begin a new paragraph to reset the language
1284                 parent_context.new_paragraph(os);
1285                 p.skip_spaces();
1286         }
1287
1288         else if (unstarred_name == "tabular" || name == "longtable") {
1289                 eat_whitespace(p, os, parent_context, false);
1290                 string width = "0pt";
1291                 if (name == "tabular*") {
1292                         width = lyx::translate_len(p.getArg('{', '}'));
1293                         eat_whitespace(p, os, parent_context, false);
1294                 }
1295                 parent_context.check_layout(os);
1296                 begin_inset(os, "Tabular ");
1297                 handle_tabular(p, os, name, width, parent_context);
1298                 end_inset(os);
1299                 p.skip_spaces();
1300         }
1301
1302         else if (parent_context.textclass.floats().typeExist(unstarred_name)) {
1303                 eat_whitespace(p, os, parent_context, false);
1304                 string const opt = p.hasOpt() ? p.getArg('[', ']') : string();
1305                 eat_whitespace(p, os, parent_context, false);
1306                 parent_context.check_layout(os);
1307                 begin_inset(os, "Float " + unstarred_name + "\n");
1308                 // store the float type for subfloats
1309                 // subfloats only work with figures and tables
1310                 if (unstarred_name == "figure")
1311                         float_type = unstarred_name;
1312                 else if (unstarred_name == "table")
1313                         float_type = unstarred_name;
1314                 else
1315                         float_type = "";
1316                 if (!opt.empty())
1317                         os << "placement " << opt << '\n';
1318                 if (contains(opt, "H"))
1319                         preamble.registerAutomaticallyLoadedPackage("float");
1320                 else {
1321                         Floating const & fl = parent_context.textclass.floats()
1322                                 .getType(unstarred_name);
1323                         if (!fl.floattype().empty() && fl.usesFloatPkg())
1324                                 preamble.registerAutomaticallyLoadedPackage("float");
1325                 }
1326
1327                 os << "wide " << convert<string>(is_starred)
1328                    << "\nsideways false"
1329                    << "\nstatus open\n\n";
1330                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1331                 end_inset(os);
1332                 // We don't need really a new paragraph, but
1333                 // we must make sure that the next item gets a \begin_layout.
1334                 parent_context.new_paragraph(os);
1335                 p.skip_spaces();
1336                 // the float is parsed thus delete the type
1337                 float_type = "";
1338         }
1339
1340         else if (unstarred_name == "sidewaysfigure"
1341                 || unstarred_name == "sidewaystable") {
1342                 eat_whitespace(p, os, parent_context, false);
1343                 parent_context.check_layout(os);
1344                 if (unstarred_name == "sidewaysfigure")
1345                         begin_inset(os, "Float figure\n");
1346                 else
1347                         begin_inset(os, "Float table\n");
1348                 os << "wide " << convert<string>(is_starred)
1349                    << "\nsideways true"
1350                    << "\nstatus open\n\n";
1351                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1352                 end_inset(os);
1353                 // We don't need really a new paragraph, but
1354                 // we must make sure that the next item gets a \begin_layout.
1355                 parent_context.new_paragraph(os);
1356                 p.skip_spaces();
1357                 preamble.registerAutomaticallyLoadedPackage("rotfloat");
1358         }
1359
1360         else if (name == "wrapfigure" || name == "wraptable") {
1361                 // syntax is \begin{wrapfigure}[lines]{placement}[overhang]{width}
1362                 eat_whitespace(p, os, parent_context, false);
1363                 parent_context.check_layout(os);
1364                 // default values
1365                 string lines = "0";
1366                 string overhang = "0col%";
1367                 // parse
1368                 if (p.hasOpt())
1369                         lines = p.getArg('[', ']');
1370                 string const placement = p.getArg('{', '}');
1371                 if (p.hasOpt())
1372                         overhang = p.getArg('[', ']');
1373                 string const width = p.getArg('{', '}');
1374                 // write
1375                 if (name == "wrapfigure")
1376                         begin_inset(os, "Wrap figure\n");
1377                 else
1378                         begin_inset(os, "Wrap table\n");
1379                 os << "lines " << lines
1380                    << "\nplacement " << placement
1381                    << "\noverhang " << lyx::translate_len(overhang)
1382                    << "\nwidth " << lyx::translate_len(width)
1383                    << "\nstatus open\n\n";
1384                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1385                 end_inset(os);
1386                 // We don't need really a new paragraph, but
1387                 // we must make sure that the next item gets a \begin_layout.
1388                 parent_context.new_paragraph(os);
1389                 p.skip_spaces();
1390                 preamble.registerAutomaticallyLoadedPackage("wrapfig");
1391         }
1392
1393         else if (name == "minipage") {
1394                 eat_whitespace(p, os, parent_context, false);
1395                 // Test whether this is an outer box of a shaded box
1396                 p.pushPosition();
1397                 // swallow arguments
1398                 while (p.hasOpt()) {
1399                         p.getArg('[', ']');
1400                         p.skip_spaces(true);
1401                 }
1402                 p.getArg('{', '}');
1403                 p.skip_spaces(true);
1404                 Token t = p.get_token();
1405                 bool shaded = false;
1406                 if (t.asInput() == "\\begin") {
1407                         p.skip_spaces(true);
1408                         if (p.getArg('{', '}') == "shaded")
1409                                 shaded = true;
1410                 }
1411                 p.popPosition();
1412                 if (shaded)
1413                         parse_outer_box(p, os, FLAG_END, outer,
1414                                         parent_context, name, "shaded");
1415                 else
1416                         parse_box(p, os, 0, FLAG_END, outer, parent_context,
1417                                   "", "", name);
1418                 p.skip_spaces();
1419         }
1420
1421         else if (name == "comment") {
1422                 eat_whitespace(p, os, parent_context, false);
1423                 parent_context.check_layout(os);
1424                 begin_inset(os, "Note Comment\n");
1425                 os << "status open\n";
1426                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1427                 end_inset(os);
1428                 p.skip_spaces();
1429                 skip_braces(p); // eat {} that might by set by LyX behind comments
1430                 preamble.registerAutomaticallyLoadedPackage("verbatim");
1431         }
1432
1433         else if (name == "verbatim") {
1434                 os << "\n\\end_layout\n\n\\begin_layout Verbatim\n";
1435                 string const s = p.plainEnvironment("verbatim");
1436                 string::const_iterator it2 = s.begin();
1437                 for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1438                         if (*it == '\\')
1439                                 os << "\\backslash ";
1440                         else if (*it == '\n') {
1441                                 it2 = it + 1;
1442                                 // avoid adding an empty paragraph at the end
1443                                 // FIXME: if there are 2 consecutive spaces at the end ignore it
1444                                 // because LyX will re-add a \n
1445                                 // This hack must be removed once bug 8049 is fixed!
1446                                 if ((it + 1 != et) && (it + 2 != et || *it2 != '\n'))
1447                                         os << "\n\\end_layout\n\\begin_layout Verbatim\n";
1448                         } else
1449                                 os << *it;
1450                 }
1451                 os << "\n\\end_layout\n\n";
1452                 p.skip_spaces();
1453                 // reset to Standard layout
1454                 os << "\n\\begin_layout Standard\n";
1455         }
1456
1457         else if (name == "CJK") {
1458                 // the scheme is \begin{CJK}{encoding}{mapping}{text}
1459                 // It is impossible to decide if a CJK environment was in its own paragraph or within
1460                 // a line. We therefore always assume a paragraph since the latter is a rare case.
1461                 eat_whitespace(p, os, parent_context, false);
1462                 parent_context.check_end_layout(os);
1463                 // store the encoding to be able to reset it
1464                 string const encoding_old = p.encoding_latex_;
1465                 string const encoding = p.getArg('{', '}');
1466                 // SJIS and BIG5 don't work with LaTeX according to the comment in unicode.cpp
1467                 // JIS does not work with LyX's encoding conversion
1468                 if (encoding != "SJIS" && encoding != "BIG5" && encoding != "JIS")
1469                         p.setEncoding(encoding);
1470                 else
1471                         p.setEncoding("utf8");
1472                 // LyX doesn't support the second argument so if
1473                 // this is used we need to output everything as ERT
1474                 string const mapping = p.getArg('{', '}');
1475                 if ( (!mapping.empty() && mapping != " ")
1476                         || (!is_known(encoding, supported_CJK_encodings))) {
1477                         parent_context.check_layout(os);
1478                         handle_ert(os, "\\begin{" + name + "}{" + encoding + "}{" + mapping + "}",
1479                                        parent_context);
1480                         // we must parse the content as verbatim because e.g. SJIS can contain
1481                         // normally invalid characters
1482                         string const s = p.plainEnvironment("CJK");
1483                         string::const_iterator it2 = s.begin();
1484                         for (string::const_iterator it = s.begin(), et = s.end(); it != et; ++it) {
1485                                 if (*it == '\\')
1486                                         handle_ert(os, "\\", parent_context);
1487                                 else if (*it == '$')
1488                                         handle_ert(os, "$", parent_context);
1489                                 else
1490                                         os << *it;
1491                         }
1492                         p.skip_spaces();
1493                         handle_ert(os, "\\end{" + name + "}",
1494                                        parent_context);
1495                 } else {
1496                         string const lang = CJK2lyx(encoding);
1497                         // store the language because we must reset it at the end
1498                         string const lang_old = parent_context.font.language;
1499                         parent_context.font.language = lang;
1500                         parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1501                         parent_context.font.language = lang_old;
1502                         parent_context.new_paragraph(os);
1503                 }
1504                 p.encoding_latex_ = encoding_old;
1505                 p.skip_spaces();
1506         }
1507
1508         else if (name == "lyxgreyedout") {
1509                 eat_whitespace(p, os, parent_context, false);
1510                 parent_context.check_layout(os);
1511                 begin_inset(os, "Note Greyedout\n");
1512                 os << "status open\n";
1513                 parse_text_in_inset(p, os, FLAG_END, outer, parent_context);
1514                 end_inset(os);
1515                 p.skip_spaces();
1516                 if (!preamble.notefontcolor().empty())
1517                         preamble.registerAutomaticallyLoadedPackage("color");
1518         }
1519
1520         else if (name == "framed" || name == "shaded") {
1521                 eat_whitespace(p, os, parent_context, false);
1522                 parse_outer_box(p, os, FLAG_END, outer, parent_context, name, "");
1523                 p.skip_spaces();
1524         }
1525
1526         else if (name == "lstlisting") {
1527                 eat_whitespace(p, os, parent_context, false);
1528                 // FIXME handle the automatic color package loading
1529                 // uwestoehr asks: In what case color is loaded?
1530                 parse_listings(p, os, parent_context, false);
1531                 p.skip_spaces();
1532         }
1533
1534         else if (!parent_context.new_layout_allowed)
1535                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1536                                           parent_context);
1537
1538         // Alignment and spacing settings
1539         // FIXME (bug xxxx): These settings can span multiple paragraphs and
1540         //                                       therefore are totally broken!
1541         // Note that \centering, raggedright, and raggedleft cannot be handled, as
1542         // they are commands not environments. They are furthermore switches that
1543         // can be ended by another switches, but also by commands like \footnote or
1544         // \parbox. So the only safe way is to leave them untouched.
1545         else if (name == "center" || name == "centering" ||
1546                  name == "flushleft" || name == "flushright" ||
1547                  name == "singlespace" || name == "onehalfspace" ||
1548                  name == "doublespace" || name == "spacing") {
1549                 eat_whitespace(p, os, parent_context, false);
1550                 // We must begin a new paragraph if not already done
1551                 if (! parent_context.atParagraphStart()) {
1552                         parent_context.check_end_layout(os);
1553                         parent_context.new_paragraph(os);
1554                 }
1555                 if (name == "flushleft")
1556                         parent_context.add_extra_stuff("\\align left\n");
1557                 else if (name == "flushright")
1558                         parent_context.add_extra_stuff("\\align right\n");
1559                 else if (name == "center" || name == "centering")
1560                         parent_context.add_extra_stuff("\\align center\n");
1561                 else if (name == "singlespace")
1562                         parent_context.add_extra_stuff("\\paragraph_spacing single\n");
1563                 else if (name == "onehalfspace") {
1564                         parent_context.add_extra_stuff("\\paragraph_spacing onehalf\n");
1565                         preamble.registerAutomaticallyLoadedPackage("setspace");
1566                 } else if (name == "doublespace") {
1567                         parent_context.add_extra_stuff("\\paragraph_spacing double\n");
1568                         preamble.registerAutomaticallyLoadedPackage("setspace");
1569                 } else if (name == "spacing") {
1570                         parent_context.add_extra_stuff("\\paragraph_spacing other " + p.verbatim_item() + "\n");
1571                         preamble.registerAutomaticallyLoadedPackage("setspace");
1572                 }
1573                 parse_text(p, os, FLAG_END, outer, parent_context);
1574                 // Just in case the environment is empty
1575                 parent_context.extra_stuff.erase();
1576                 // We must begin a new paragraph to reset the alignment
1577                 parent_context.new_paragraph(os);
1578                 p.skip_spaces();
1579         }
1580
1581         // The single '=' is meant here.
1582         else if ((newlayout = findLayout(parent_context.textclass, name, false))) {
1583                 eat_whitespace(p, os, parent_context, false);
1584                 Context context(true, parent_context.textclass, newlayout,
1585                                 parent_context.layout, parent_context.font);
1586                 if (parent_context.deeper_paragraph) {
1587                         // We are beginning a nested environment after a
1588                         // deeper paragraph inside the outer list environment.
1589                         // Therefore we don't need to output a "begin deeper".
1590                         context.need_end_deeper = true;
1591                 }
1592                 parent_context.check_end_layout(os);
1593                 if (last_env == name) {
1594                         // we need to output a separator since LyX would export
1595                         // the two environments as one otherwise (bug 5716)
1596                         docstring const sep = from_ascii("--Separator--");
1597                         TeX2LyXDocClass const & textclass(parent_context.textclass);
1598                         if (textclass.hasLayout(sep)) {
1599                                 Context newcontext(parent_context);
1600                                 newcontext.layout = &(textclass[sep]);
1601                                 newcontext.check_layout(os);
1602                                 newcontext.check_end_layout(os);
1603                         } else {
1604                                 parent_context.check_layout(os);
1605                                 begin_inset(os, "Note Note\n");
1606                                 os << "status closed\n";
1607                                 Context newcontext(true, textclass,
1608                                                 &(textclass.defaultLayout()));
1609                                 newcontext.check_layout(os);
1610                                 newcontext.check_end_layout(os);
1611                                 end_inset(os);
1612                                 parent_context.check_end_layout(os);
1613                         }
1614                 }
1615                 switch (context.layout->latextype) {
1616                 case  LATEX_LIST_ENVIRONMENT:
1617                         context.add_par_extra_stuff("\\labelwidthstring "
1618                                                     + p.verbatim_item() + '\n');
1619                         p.skip_spaces();
1620                         break;
1621                 case  LATEX_BIB_ENVIRONMENT:
1622                         p.verbatim_item(); // swallow next arg
1623                         p.skip_spaces();
1624                         break;
1625                 default:
1626                         break;
1627                 }
1628                 context.check_deeper(os);
1629                 // handle known optional and required arguments
1630                 // layouts require all optional arguments before the required ones
1631                 // Unfortunately LyX can't handle arguments of list arguments (bug 7468):
1632                 // It is impossible to place anything after the environment name,
1633                 // but before the first \\item.
1634                 if (context.layout->latextype == LATEX_ENVIRONMENT) {
1635                         bool need_layout = true;
1636                         unsigned int optargs = 0;
1637                         while (optargs < context.layout->optargs) {
1638                                 eat_whitespace(p, os, context, false);
1639                                 if (p.next_token().cat() == catEscape ||
1640                                     p.next_token().character() != '[')
1641                                         break;
1642                                 p.get_token(); // eat '['
1643                                 if (need_layout) {
1644                                         context.check_layout(os);
1645                                         need_layout = false;
1646                                 }
1647                                 begin_inset(os, "Argument\n");
1648                                 os << "status collapsed\n\n";
1649                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
1650                                 end_inset(os);
1651                                 eat_whitespace(p, os, context, false);
1652                                 ++optargs;
1653                         }
1654                         unsigned int reqargs = 0;
1655                         while (reqargs < context.layout->reqargs) {
1656                                 eat_whitespace(p, os, context, false);
1657                                 if (p.next_token().cat() != catBegin)
1658                                         break;
1659                                 p.get_token(); // eat '{'
1660                                 if (need_layout) {
1661                                         context.check_layout(os);
1662                                         need_layout = false;
1663                                 }
1664                                 begin_inset(os, "Argument\n");
1665                                 os << "status collapsed\n\n";
1666                                 parse_text_in_inset(p, os, FLAG_BRACE_LAST, outer, context);
1667                                 end_inset(os);
1668                                 eat_whitespace(p, os, context, false);
1669                                 ++reqargs;
1670                         }
1671                 }
1672                 parse_text(p, os, FLAG_END, outer, context);
1673                 context.check_end_layout(os);
1674                 if (parent_context.deeper_paragraph) {
1675                         // We must suppress the "end deeper" because we
1676                         // suppressed the "begin deeper" above.
1677                         context.need_end_deeper = false;
1678                 }
1679                 context.check_end_deeper(os);
1680                 parent_context.new_paragraph(os);
1681                 p.skip_spaces();
1682                 if (!preamble.titleLayoutFound())
1683                         preamble.titleLayoutFound(newlayout->intitle);
1684                 set<string> const & req = newlayout->requires();
1685                 set<string>::const_iterator it = req.begin();
1686                 set<string>::const_iterator en = req.end();
1687                 for (; it != en; ++it)
1688                         preamble.registerAutomaticallyLoadedPackage(*it);
1689         }
1690
1691         // The single '=' is meant here.
1692         else if ((newinsetlayout = findInsetLayout(parent_context.textclass, name, false))) {
1693                 eat_whitespace(p, os, parent_context, false);
1694                 parent_context.check_layout(os);
1695                 begin_inset(os, "Flex ");
1696                 os << to_utf8(newinsetlayout->name()) << '\n'
1697                    << "status collapsed\n";
1698                 parse_text_in_inset(p, os, FLAG_END, false, parent_context, newinsetlayout);
1699                 end_inset(os);
1700         }
1701
1702         else if (name == "appendix") {
1703                 // This is no good latex style, but it works and is used in some documents...
1704                 eat_whitespace(p, os, parent_context, false);
1705                 parent_context.check_end_layout(os);
1706                 Context context(true, parent_context.textclass, parent_context.layout,
1707                                 parent_context.layout, parent_context.font);
1708                 context.check_layout(os);
1709                 os << "\\start_of_appendix\n";
1710                 parse_text(p, os, FLAG_END, outer, context);
1711                 context.check_end_layout(os);
1712                 p.skip_spaces();
1713         }
1714
1715         else if (known_environments.find(name) != known_environments.end()) {
1716                 vector<ArgumentType> arguments = known_environments[name];
1717                 // The last "argument" denotes wether we may translate the
1718                 // environment contents to LyX
1719                 // The default required if no argument is given makes us
1720                 // compatible with the reLyXre environment.
1721                 ArgumentType contents = arguments.empty() ?
1722                         required :
1723                         arguments.back();
1724                 if (!arguments.empty())
1725                         arguments.pop_back();
1726                 // See comment in parse_unknown_environment()
1727                 bool const specialfont =
1728                         (parent_context.font != parent_context.normalfont);
1729                 bool const new_layout_allowed =
1730                         parent_context.new_layout_allowed;
1731                 if (specialfont)
1732                         parent_context.new_layout_allowed = false;
1733                 parse_arguments("\\begin{" + name + "}", arguments, p, os,
1734                                 outer, parent_context);
1735                 if (contents == verbatim)
1736                         handle_ert(os, p.verbatimEnvironment(name),
1737                                    parent_context);
1738                 else
1739                         parse_text_snippet(p, os, FLAG_END, outer,
1740                                            parent_context);
1741                 handle_ert(os, "\\end{" + name + "}", parent_context);
1742                 if (specialfont)
1743                         parent_context.new_layout_allowed = new_layout_allowed;
1744         }
1745
1746         else
1747                 parse_unknown_environment(p, name, os, FLAG_END, outer,
1748                                           parent_context);
1749
1750         last_env = name;
1751         active_environments.pop_back();
1752 }
1753
1754
1755 /// parses a comment and outputs it to \p os.
1756 void parse_comment(Parser & p, ostream & os, Token const & t, Context & context)
1757 {
1758         LASSERT(t.cat() == catComment, return);
1759         if (!t.cs().empty()) {
1760                 context.check_layout(os);
1761                 handle_comment(os, '%' + t.cs(), context);
1762                 if (p.next_token().cat() == catNewline) {
1763                         // A newline after a comment line starts a new
1764                         // paragraph
1765                         if (context.new_layout_allowed) {
1766                                 if(!context.atParagraphStart())
1767                                         // Only start a new paragraph if not already
1768                                         // done (we might get called recursively)
1769                                         context.new_paragraph(os);
1770                         } else
1771                                 handle_ert(os, "\n", context);
1772                         eat_whitespace(p, os, context, true);
1773                 }
1774         } else {
1775                 // "%\n" combination
1776                 p.skip_spaces();
1777         }
1778 }
1779
1780
1781 /*!
1782  * Reads spaces and comments until the first non-space, non-comment token.
1783  * New paragraphs (double newlines or \\par) are handled like simple spaces
1784  * if \p eatParagraph is true.
1785  * Spaces are skipped, but comments are written to \p os.
1786  */
1787 void eat_whitespace(Parser & p, ostream & os, Context & context,
1788                     bool eatParagraph)
1789 {
1790         while (p.good()) {
1791                 Token const & t = p.get_token();
1792                 if (t.cat() == catComment)
1793                         parse_comment(p, os, t, context);
1794                 else if ((! eatParagraph && p.isParagraph()) ||
1795                          (t.cat() != catSpace && t.cat() != catNewline)) {
1796                         p.putback();
1797                         return;
1798                 }
1799         }
1800 }
1801
1802
1803 /*!
1804  * Set a font attribute, parse text and reset the font attribute.
1805  * \param attribute Attribute name (e.g. \\family, \\shape etc.)
1806  * \param currentvalue Current value of the attribute. Is set to the new
1807  * value during parsing.
1808  * \param newvalue New value of the attribute
1809  */
1810 void parse_text_attributes(Parser & p, ostream & os, unsigned flags, bool outer,
1811                            Context & context, string const & attribute,
1812                            string & currentvalue, string const & newvalue)
1813 {
1814         context.check_layout(os);
1815         string const oldvalue = currentvalue;
1816         currentvalue = newvalue;
1817         os << '\n' << attribute << ' ' << newvalue << "\n";
1818         parse_text_snippet(p, os, flags, outer, context);
1819         context.check_layout(os);
1820         os << '\n' << attribute << ' ' << oldvalue << "\n";
1821         currentvalue = oldvalue;
1822 }
1823
1824
1825 /// get the arguments of a natbib or jurabib citation command
1826 void get_cite_arguments(Parser & p, bool natbibOrder,
1827         string & before, string & after)
1828 {
1829         // We need to distinguish "" and "[]", so we can't use p.getOpt().
1830
1831         // text before the citation
1832         before.clear();
1833         // text after the citation
1834         after = p.getFullOpt();
1835
1836         if (!after.empty()) {
1837                 before = p.getFullOpt();
1838                 if (natbibOrder && !before.empty())
1839                         swap(before, after);
1840         }
1841 }
1842
1843
1844 /// Convert filenames with TeX macros and/or quotes to something LyX
1845 /// can understand
1846 string const normalize_filename(string const & name)
1847 {
1848         Parser p(trim(name, "\""));
1849         ostringstream os;
1850         while (p.good()) {
1851                 Token const & t = p.get_token();
1852                 if (t.cat() != catEscape)
1853                         os << t.asInput();
1854                 else if (t.cs() == "lyxdot") {
1855                         // This is used by LyX for simple dots in relative
1856                         // names
1857                         os << '.';
1858                         p.skip_spaces();
1859                 } else if (t.cs() == "space") {
1860                         os << ' ';
1861                         p.skip_spaces();
1862                 } else
1863                         os << t.asInput();
1864         }
1865         return os.str();
1866 }
1867
1868
1869 /// Convert \p name from TeX convention (relative to master file) to LyX
1870 /// convention (relative to .lyx file) if it is relative
1871 void fix_relative_filename(string & name)
1872 {
1873         if (FileName::isAbsolute(name))
1874                 return;
1875
1876         name = to_utf8(makeRelPath(from_utf8(makeAbsPath(name, getMasterFilePath()).absFileName()),
1877                                    from_utf8(getParentFilePath())));
1878 }
1879
1880
1881 /// Parse a NoWeb Scrap section. The initial "<<" is already parsed.
1882 void parse_noweb(Parser & p, ostream & os, Context & context)
1883 {
1884         // assemble the rest of the keyword
1885         string name("<<");
1886         bool scrap = false;
1887         while (p.good()) {
1888                 Token const & t = p.get_token();
1889                 if (t.asInput() == ">" && p.next_token().asInput() == ">") {
1890                         name += ">>";
1891                         p.get_token();
1892                         scrap = (p.good() && p.next_token().asInput() == "=");
1893                         if (scrap)
1894                                 name += p.get_token().asInput();
1895                         break;
1896                 }
1897                 name += t.asInput();
1898         }
1899
1900         if (!scrap || !context.new_layout_allowed ||
1901             !context.textclass.hasLayout(from_ascii("Scrap"))) {
1902                 cerr << "Warning: Could not interpret '" << name
1903                      << "'. Ignoring it." << endl;
1904                 return;
1905         }
1906
1907         // We use new_paragraph instead of check_end_layout because the stuff
1908         // following the noweb chunk needs to start with a \begin_layout.
1909         // This may create a new paragraph even if there was none in the
1910         // noweb file, but the alternative is an invalid LyX file. Since
1911         // noweb code chunks are implemented with a layout style in LyX they
1912         // always must be in an own paragraph.
1913         context.new_paragraph(os);
1914         Context newcontext(true, context.textclass,
1915                 &context.textclass[from_ascii("Scrap")]);
1916         newcontext.check_layout(os);
1917         os << name;
1918         while (p.good()) {
1919                 Token const & t = p.get_token();
1920                 // We abuse the parser a bit, because this is no TeX syntax
1921                 // at all.
1922                 if (t.cat() == catEscape)
1923                         os << subst(t.asInput(), "\\", "\n\\backslash\n");
1924                 else {
1925                         ostringstream oss;
1926                         Context tmp(false, context.textclass,
1927                                     &context.textclass[from_ascii("Scrap")]);
1928                         tmp.need_end_layout = true;
1929                         tmp.check_layout(oss);
1930                         os << subst(t.asInput(), "\n", oss.str());
1931                 }
1932                 // The scrap chunk is ended by an @ at the beginning of a line.
1933                 // After the @ the line may contain a comment and/or
1934                 // whitespace, but nothing else.
1935                 if (t.asInput() == "@" && p.prev_token().cat() == catNewline &&
1936                     (p.next_token().cat() == catSpace ||
1937                      p.next_token().cat() == catNewline ||
1938                      p.next_token().cat() == catComment)) {
1939                         while (p.good() && p.next_token().cat() == catSpace)
1940                                 os << p.get_token().asInput();
1941                         if (p.next_token().cat() == catComment)
1942                                 // The comment includes a final '\n'
1943                                 os << p.get_token().asInput();
1944                         else {
1945                                 if (p.next_token().cat() == catNewline)
1946                                         p.get_token();
1947                                 os << '\n';
1948                         }
1949                         break;
1950                 }
1951         }
1952         newcontext.check_end_layout(os);
1953 }
1954
1955
1956 /// detects \\def, \\long\\def and \\global\\long\\def with ws and comments
1957 bool is_macro(Parser & p)
1958 {
1959         Token first = p.curr_token();
1960         if (first.cat() != catEscape || !p.good())
1961                 return false;
1962         if (first.cs() == "def")
1963                 return true;
1964         if (first.cs() != "global" && first.cs() != "long")
1965                 return false;
1966         Token second = p.get_token();
1967         int pos = 1;
1968         while (p.good() && !p.isParagraph() && (second.cat() == catSpace ||
1969                second.cat() == catNewline || second.cat() == catComment)) {
1970                 second = p.get_token();
1971                 pos++;
1972         }
1973         bool secondvalid = second.cat() == catEscape;
1974         Token third;
1975         bool thirdvalid = false;
1976         if (p.good() && first.cs() == "global" && secondvalid &&
1977             second.cs() == "long") {
1978                 third = p.get_token();
1979                 pos++;
1980                 while (p.good() && !p.isParagraph() &&
1981                        (third.cat() == catSpace ||
1982                         third.cat() == catNewline ||
1983                         third.cat() == catComment)) {
1984                         third = p.get_token();
1985                         pos++;
1986                 }
1987                 thirdvalid = third.cat() == catEscape;
1988         }
1989         for (int i = 0; i < pos; ++i)
1990                 p.putback();
1991         if (!secondvalid)
1992                 return false;
1993         if (!thirdvalid)
1994                 return (first.cs() == "global" || first.cs() == "long") &&
1995                        second.cs() == "def";
1996         return first.cs() == "global" && second.cs() == "long" &&
1997                third.cs() == "def";
1998 }
1999
2000
2001 /// Parse a macro definition (assumes that is_macro() returned true)
2002 void parse_macro(Parser & p, ostream & os, Context & context)
2003 {
2004         context.check_layout(os);
2005         Token first = p.curr_token();
2006         Token second;
2007         Token third;
2008         string command = first.asInput();
2009         if (first.cs() != "def") {
2010                 p.get_token();
2011                 eat_whitespace(p, os, context, false);
2012                 second = p.curr_token();
2013                 command += second.asInput();
2014                 if (second.cs() != "def") {
2015                         p.get_token();
2016                         eat_whitespace(p, os, context, false);
2017                         third = p.curr_token();
2018                         command += third.asInput();
2019                 }
2020         }
2021         eat_whitespace(p, os, context, false);
2022         string const name = p.get_token().cs();
2023         eat_whitespace(p, os, context, false);
2024
2025         // parameter text
2026         bool simple = true;
2027         string paramtext;
2028         int arity = 0;
2029         while (p.next_token().cat() != catBegin) {
2030                 if (p.next_token().cat() == catParameter) {
2031                         // # found
2032                         p.get_token();
2033                         paramtext += "#";
2034
2035                         // followed by number?
2036                         if (p.next_token().cat() == catOther) {
2037                                 char c = p.getChar();
2038                                 paramtext += c;
2039                                 // number = current arity + 1?
2040                                 if (c == arity + '0' + 1)
2041                                         ++arity;
2042                                 else
2043                                         simple = false;
2044                         } else
2045                                 paramtext += p.get_token().cs();
2046                 } else {
2047                         paramtext += p.get_token().cs();
2048                         simple = false;
2049                 }
2050         }
2051
2052         // only output simple (i.e. compatible) macro as FormulaMacros
2053         string ert = '\\' + name + ' ' + paramtext + '{' + p.verbatim_item() + '}';
2054         if (simple) {
2055                 context.check_layout(os);
2056                 begin_inset(os, "FormulaMacro");
2057                 os << "\n\\def" << ert;
2058                 end_inset(os);
2059         } else
2060                 handle_ert(os, command + ert, context);
2061 }
2062
2063
2064 void registerExternalTemplatePackages(string const & name)
2065 {
2066         external::TemplateManager const & etm = external::TemplateManager::get();
2067         external::Template const * const et = etm.getTemplateByName(name);
2068         if (!et)
2069                 return;
2070         external::Template::Formats::const_iterator cit = et->formats.end();
2071         if (pdflatex)
2072                 cit = et->formats.find("PDFLaTeX");
2073         if (cit == et->formats.end())
2074                 // If the template has not specified a PDFLaTeX output,
2075                 // we try the LaTeX format.
2076                 cit = et->formats.find("LaTeX");
2077         if (cit == et->formats.end())
2078                 return;
2079         vector<string>::const_iterator qit = cit->second.requirements.begin();
2080         vector<string>::const_iterator qend = cit->second.requirements.end();
2081         for (; qit != qend; ++qit)
2082                 preamble.registerAutomaticallyLoadedPackage(*qit);
2083 }
2084
2085 } // anonymous namespace
2086
2087
2088 void parse_text(Parser & p, ostream & os, unsigned flags, bool outer,
2089                 Context & context)
2090 {
2091         Layout const * newlayout = 0;
2092         InsetLayout const * newinsetlayout = 0;
2093         char const * const * where = 0;
2094         // Store the latest bibliographystyle and nocite{*} option
2095         // (needed for bibtex inset)
2096         string btprint;
2097         string bibliographystyle = "default";
2098         bool const use_natbib = preamble.isPackageUsed("natbib");
2099         bool const use_jurabib = preamble.isPackageUsed("jurabib");
2100         string last_env;
2101         while (p.good()) {
2102                 Token const & t = p.get_token();
2103
2104         // it is impossible to determine the correct document language if CJK is used.
2105         // Therefore write a note at the beginning of the document
2106         if (have_CJK) {
2107                 context.check_layout(os);
2108                 begin_inset(os, "Note Note\n");
2109                 os << "status open\n\\begin_layout Plain Layout\n"
2110                    << "\\series bold\n"
2111                    << "Important information:\n"
2112                    << "\\end_layout\n\n"
2113                    << "\\begin_layout Plain Layout\n"
2114                    << "This document contains text in Chinese, Japanese or Korean.\n"
2115                    << " It was therefore impossible for tex2lyx to set the correct document langue for your document."
2116                    << " Please set in the document settings by yourself!\n"
2117                    << "\\end_layout\n";
2118                 end_inset(os);
2119                 have_CJK = false;
2120         }
2121
2122 #ifdef FILEDEBUG
2123                 debugToken(cerr, t, flags);
2124 #endif
2125
2126                 if (flags & FLAG_ITEM) {
2127                         if (t.cat() == catSpace)
2128                                 continue;
2129
2130                         flags &= ~FLAG_ITEM;
2131                         if (t.cat() == catBegin) {
2132                                 // skip the brace and collect everything to the next matching
2133                                 // closing brace
2134                                 flags |= FLAG_BRACE_LAST;
2135                                 continue;
2136                         }
2137
2138                         // handle only this single token, leave the loop if done
2139                         flags |= FLAG_LEAVE;
2140                 }
2141
2142                 if (t.cat() != catEscape && t.character() == ']' &&
2143                     (flags & FLAG_BRACK_LAST))
2144                         return;
2145                 if (t.cat() == catEnd && (flags & FLAG_BRACE_LAST))
2146                         return;
2147
2148                 // If there is anything between \end{env} and \begin{env} we
2149                 // don't need to output a separator.
2150                 if (t.cat() != catSpace && t.cat() != catNewline &&
2151                     t.asInput() != "\\begin")
2152                         last_env = "";
2153
2154                 //
2155                 // cat codes
2156                 //
2157                 if (t.cat() == catMath) {
2158                         // we are inside some text mode thingy, so opening new math is allowed
2159                         context.check_layout(os);
2160                         begin_inset(os, "Formula ");
2161                         Token const & n = p.get_token();
2162                         bool const display(n.cat() == catMath && outer);
2163                         if (display) {
2164                                 // TeX's $$...$$ syntax for displayed math
2165                                 os << "\\[";
2166                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2167                                 os << "\\]";
2168                                 p.get_token(); // skip the second '$' token
2169                         } else {
2170                                 // simple $...$  stuff
2171                                 p.putback();
2172                                 os << '$';
2173                                 parse_math(p, os, FLAG_SIMPLE, MATH_MODE);
2174                                 os << '$';
2175                         }
2176                         end_inset(os);
2177                         if (display) {
2178                                 // Prevent the conversion of a line break to a
2179                                 // space (bug 7668). This does not change the
2180                                 // output, but looks ugly in LyX.
2181                                 eat_whitespace(p, os, context, false);
2182                         }
2183                 }
2184
2185                 else if (t.cat() == catSuper || t.cat() == catSub)
2186                         cerr << "catcode " << t << " illegal in text mode\n";
2187
2188                 // Basic support for english quotes. This should be
2189                 // extended to other quotes, but is not so easy (a
2190                 // left english quote is the same as a right german
2191                 // quote...)
2192                 else if (t.asInput() == "`" && p.next_token().asInput() == "`") {
2193                         context.check_layout(os);
2194                         begin_inset(os, "Quotes ");
2195                         os << "eld";
2196                         end_inset(os);
2197                         p.get_token();
2198                         skip_braces(p);
2199                 }
2200                 else if (t.asInput() == "'" && p.next_token().asInput() == "'") {
2201                         context.check_layout(os);
2202                         begin_inset(os, "Quotes ");
2203                         os << "erd";
2204                         end_inset(os);
2205                         p.get_token();
2206                         skip_braces(p);
2207                 }
2208
2209                 else if (t.asInput() == ">" && p.next_token().asInput() == ">") {
2210                         context.check_layout(os);
2211                         begin_inset(os, "Quotes ");
2212                         os << "ald";
2213                         end_inset(os);
2214                         p.get_token();
2215                         skip_braces(p);
2216                 }
2217
2218                 else if (t.asInput() == "<" && p.next_token().asInput() == "<") {
2219                         context.check_layout(os);
2220                         begin_inset(os, "Quotes ");
2221                         os << "ard";
2222                         end_inset(os);
2223                         p.get_token();
2224                         skip_braces(p);
2225                 }
2226
2227                 else if (t.asInput() == "<"
2228                          && p.next_token().asInput() == "<" && noweb_mode) {
2229                         p.get_token();
2230                         parse_noweb(p, os, context);
2231                 }
2232
2233                 else if (t.cat() == catSpace || (t.cat() == catNewline && ! p.isParagraph()))
2234                         check_space(p, os, context);
2235
2236                 else if (t.character() == '[' && noweb_mode &&
2237                          p.next_token().character() == '[') {
2238                         // These can contain underscores
2239                         p.putback();
2240                         string const s = p.getFullOpt() + ']';
2241                         if (p.next_token().character() == ']')
2242                                 p.get_token();
2243                         else
2244                                 cerr << "Warning: Inserting missing ']' in '"
2245                                      << s << "'." << endl;
2246                         handle_ert(os, s, context);
2247                 }
2248
2249                 else if (t.cat() == catLetter) {
2250                         context.check_layout(os);
2251                         // Workaround for bug 4752.
2252                         // FIXME: This whole code block needs to be removed
2253                         //        when the bug is fixed and tex2lyx produces
2254                         //        the updated file format.
2255                         // The replacement algorithm in LyX is so stupid that
2256                         // it even translates a phrase if it is part of a word.
2257                         bool handled = false;
2258                         for (int const * l = known_phrase_lengths; *l; ++l) {
2259                                 string phrase = t.cs();
2260                                 for (int i = 1; i < *l && p.next_token().isAlnumASCII(); ++i)
2261                                         phrase += p.get_token().cs();
2262                                 if (is_known(phrase, known_coded_phrases)) {
2263                                         handle_ert(os, phrase, context);
2264                                         handled = true;
2265                                         break;
2266                                 } else {
2267                                         for (size_t i = 1; i < phrase.length(); ++i)
2268                                                 p.putback();
2269                                 }
2270                         }
2271                         if (!handled)
2272                                 os << t.cs();
2273                 }
2274
2275                 else if (t.cat() == catOther ||
2276                                t.cat() == catAlign ||
2277                                t.cat() == catParameter) {
2278                         // This translates "&" to "\\&" which may be wrong...
2279                         context.check_layout(os);
2280                         os << t.cs();
2281                 }
2282
2283                 else if (p.isParagraph()) {
2284                         if (context.new_layout_allowed)
2285                                 context.new_paragraph(os);
2286                         else
2287                                 handle_ert(os, "\\par ", context);
2288                         eat_whitespace(p, os, context, true);
2289                 }
2290
2291                 else if (t.cat() == catActive) {
2292                         context.check_layout(os);
2293                         if (t.character() == '~') {
2294                                 if (context.layout->free_spacing)
2295                                         os << ' ';
2296                                 else {
2297                                         begin_inset(os, "space ~\n");
2298                                         end_inset(os);
2299                                 }
2300                         } else
2301                                 os << t.cs();
2302                 }
2303
2304                 else if (t.cat() == catBegin) {
2305                         Token const next = p.next_token();
2306                         Token const end = p.next_next_token();
2307                         if (next.cat() == catEnd) {
2308                         // {}
2309                         Token const prev = p.prev_token();
2310                         p.get_token();
2311                         if (p.next_token().character() == '`' ||
2312                             (prev.character() == '-' &&
2313                              p.next_token().character() == '-'))
2314                                 ; // ignore it in {}`` or -{}-
2315                         else
2316                                 handle_ert(os, "{}", context);
2317                         } else if (next.cat() == catEscape &&
2318                                    is_known(next.cs(), known_quotes) &&
2319                                    end.cat() == catEnd) {
2320                                 // Something like {\textquoteright} (e.g.
2321                                 // from writer2latex). LyX writes
2322                                 // \textquoteright{}, so we may skip the
2323                                 // braces here for better readability.
2324                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2325                                                    outer, context);
2326                         } else {
2327                         context.check_layout(os);
2328                         // special handling of font attribute changes
2329                         Token const prev = p.prev_token();
2330                         TeXFont const oldFont = context.font;
2331                         if (next.character() == '[' ||
2332                             next.character() == ']' ||
2333                             next.character() == '*') {
2334                                 p.get_token();
2335                                 if (p.next_token().cat() == catEnd) {
2336                                         os << next.cs();
2337                                         p.get_token();
2338                                 } else {
2339                                         p.putback();
2340                                         handle_ert(os, "{", context);
2341                                         parse_text_snippet(p, os,
2342                                                         FLAG_BRACE_LAST,
2343                                                         outer, context);
2344                                         handle_ert(os, "}", context);
2345                                 }
2346                         } else if (! context.new_layout_allowed) {
2347                                 handle_ert(os, "{", context);
2348                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2349                                                    outer, context);
2350                                 handle_ert(os, "}", context);
2351                         } else if (is_known(next.cs(), known_sizes)) {
2352                                 // next will change the size, so we must
2353                                 // reset it here
2354                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2355                                                    outer, context);
2356                                 if (!context.atParagraphStart())
2357                                         os << "\n\\size "
2358                                            << context.font.size << "\n";
2359                         } else if (is_known(next.cs(), known_font_families)) {
2360                                 // next will change the font family, so we
2361                                 // must reset it here
2362                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2363                                                    outer, context);
2364                                 if (!context.atParagraphStart())
2365                                         os << "\n\\family "
2366                                            << context.font.family << "\n";
2367                         } else if (is_known(next.cs(), known_font_series)) {
2368                                 // next will change the font series, so we
2369                                 // must reset it here
2370                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2371                                                    outer, context);
2372                                 if (!context.atParagraphStart())
2373                                         os << "\n\\series "
2374                                            << context.font.series << "\n";
2375                         } else if (is_known(next.cs(), known_font_shapes)) {
2376                                 // next will change the font shape, so we
2377                                 // must reset it here
2378                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2379                                                    outer, context);
2380                                 if (!context.atParagraphStart())
2381                                         os << "\n\\shape "
2382                                            << context.font.shape << "\n";
2383                         } else if (is_known(next.cs(), known_old_font_families) ||
2384                                    is_known(next.cs(), known_old_font_series) ||
2385                                    is_known(next.cs(), known_old_font_shapes)) {
2386                                 // next will change the font family, series
2387                                 // and shape, so we must reset it here
2388                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2389                                                    outer, context);
2390                                 if (!context.atParagraphStart())
2391                                         os <<  "\n\\family "
2392                                            << context.font.family
2393                                            << "\n\\series "
2394                                            << context.font.series
2395                                            << "\n\\shape "
2396                                            << context.font.shape << "\n";
2397                         } else {
2398                                 handle_ert(os, "{", context);
2399                                 parse_text_snippet(p, os, FLAG_BRACE_LAST,
2400                                                    outer, context);
2401                                 handle_ert(os, "}", context);
2402                                 }
2403                         }
2404                 }
2405
2406                 else if (t.cat() == catEnd) {
2407                         if (flags & FLAG_BRACE_LAST) {
2408                                 return;
2409                         }
2410                         cerr << "stray '}' in text\n";
2411                         handle_ert(os, "}", context);
2412                 }
2413
2414                 else if (t.cat() == catComment)
2415                         parse_comment(p, os, t, context);
2416
2417                 //
2418                 // control sequences
2419                 //
2420
2421                 else if (t.cs() == "(") {
2422                         context.check_layout(os);
2423                         begin_inset(os, "Formula");
2424                         os << " \\(";
2425                         parse_math(p, os, FLAG_SIMPLE2, MATH_MODE);
2426                         os << "\\)";
2427                         end_inset(os);
2428                 }
2429
2430                 else if (t.cs() == "[") {
2431                         context.check_layout(os);
2432                         begin_inset(os, "Formula");
2433                         os << " \\[";
2434                         parse_math(p, os, FLAG_EQUATION, MATH_MODE);
2435                         os << "\\]";
2436                         end_inset(os);
2437                         // Prevent the conversion of a line break to a space
2438                         // (bug 7668). This does not change the output, but
2439                         // looks ugly in LyX.
2440                         eat_whitespace(p, os, context, false);
2441                 }
2442
2443                 else if (t.cs() == "begin")
2444                         parse_environment(p, os, outer, last_env,
2445                                           context);
2446
2447                 else if (t.cs() == "end") {
2448                         if (flags & FLAG_END) {
2449                                 // eat environment name
2450                                 string const name = p.getArg('{', '}');
2451                                 if (name != active_environment())
2452                                         cerr << "\\end{" + name + "} does not match \\begin{"
2453                                                 + active_environment() + "}\n";
2454                                 return;
2455                         }
2456                         p.error("found 'end' unexpectedly");
2457                 }
2458
2459                 else if (t.cs() == "item") {
2460                         string s;
2461                         bool const optarg = p.hasOpt();
2462                         if (optarg) {
2463                                 // FIXME: This swallows comments, but we cannot use
2464                                 //        eat_whitespace() since we must not output
2465                                 //        anything before the item.
2466                                 p.skip_spaces(true);
2467                                 s = p.verbatimOption();
2468                         } else
2469                                 p.skip_spaces(false);
2470                         context.set_item();
2471                         context.check_layout(os);
2472                         if (context.has_item) {
2473                                 // An item in an unknown list-like environment
2474                                 // FIXME: Do this in check_layout()!
2475                                 context.has_item = false;
2476                                 if (optarg)
2477                                         handle_ert(os, "\\item", context);
2478                                 else
2479                                         handle_ert(os, "\\item ", context);
2480                         }
2481                         if (optarg) {
2482                                 if (context.layout->labeltype != LABEL_MANUAL) {
2483                                         // LyX does not support \item[\mybullet]
2484                                         // in itemize environments
2485                                         Parser p2(s + ']');
2486                                         os << parse_text_snippet(p2,
2487                                                 FLAG_BRACK_LAST, outer, context);
2488                                 } else if (!s.empty()) {
2489                                         // LyX adds braces around the argument,
2490                                         // so we need to remove them here.
2491                                         if (s.size() > 2 && s[0] == '{' &&
2492                                             s[s.size()-1] == '}')
2493                                                 s = s.substr(1, s.size()-2);
2494                                         // If the argument contains a space we
2495                                         // must put it into ERT: Otherwise LyX
2496                                         // would misinterpret the space as
2497                                         // item delimiter (bug 7663)
2498                                         if (contains(s, ' ')) {
2499                                                 handle_ert(os, s, context);
2500                                         } else {
2501                                                 Parser p2(s + ']');
2502                                                 os << parse_text_snippet(p2,
2503                                                         FLAG_BRACK_LAST,
2504                                                         outer, context);
2505                                         }
2506                                         // The space is needed to separate the
2507                                         // item from the rest of the sentence.
2508                                         os << ' ';
2509                                         eat_whitespace(p, os, context, false);
2510                                 }
2511                         }
2512                 }
2513
2514                 else if (t.cs() == "bibitem") {
2515                         context.set_item();
2516                         context.check_layout(os);
2517                         eat_whitespace(p, os, context, false);
2518                         string label = convert_command_inset_arg(p.verbatimOption());
2519                         string key = convert_command_inset_arg(p.verbatim_item());
2520                         if (contains(label, '\\') || contains(key, '\\')) {
2521                                 // LyX can't handle LaTeX commands in labels or keys
2522                                 handle_ert(os, t.asInput() + '[' + label +
2523                                                "]{" + p.verbatim_item() + '}',
2524                                            context);
2525                         } else {
2526                                 begin_command_inset(os, "bibitem", "bibitem");
2527                                 os << "label \"" << label << "\"\n"
2528                                       "key \"" << key << "\"\n";
2529                                 end_inset(os);
2530                         }
2531                 }
2532
2533                 else if (is_macro(p)) {
2534                         // catch the case of \def\inputGnumericTable
2535                         bool macro = true;
2536                         if (t.cs() == "def") {
2537                                 Token second = p.next_token();
2538                                 if (second.cs() == "inputGnumericTable") {
2539                                         p.pushPosition();
2540                                         p.get_token();
2541                                         skip_braces(p);
2542                                         Token third = p.get_token();
2543                                         p.popPosition();
2544                                         if (third.cs() == "input") {
2545                                                 p.get_token();
2546                                                 skip_braces(p);
2547                                                 p.get_token();
2548                                                 string name = normalize_filename(p.verbatim_item());
2549                                                 string const path = getMasterFilePath();
2550                                                 // We want to preserve relative / absolute filenames,
2551                                                 // therefore path is only used for testing
2552                                                 // The file extension is in every case ".tex".
2553                                                 // So we need to remove this extension and check for
2554                                                 // the original one.
2555                                                 name = removeExtension(name);
2556                                                 if (!makeAbsPath(name, path).exists()) {
2557                                                         char const * const Gnumeric_formats[] = {"gnumeric",
2558                                                                 "ods", "xls", 0};
2559                                                         string const Gnumeric_name =
2560                                                                 find_file(name, path, Gnumeric_formats);
2561                                                         if (!Gnumeric_name.empty())
2562                                                                 name = Gnumeric_name;
2563                                                 }
2564                                                 if (makeAbsPath(name, path).exists())
2565                                                         fix_relative_filename(name);
2566                                                 else
2567                                                         cerr << "Warning: Could not find file '"
2568                                                              << name << "'." << endl;
2569                                                 context.check_layout(os);
2570                                                 begin_inset(os, "External\n\ttemplate ");
2571                                                 os << "GnumericSpreadsheet\n\tfilename "
2572                                                    << name << "\n";
2573                                                 end_inset(os);
2574                                                 context.check_layout(os);
2575                                                 macro = false;
2576                                                 // register the packages that are automatically reloaded
2577                                                 // by the Gnumeric template
2578                                                 registerExternalTemplatePackages("GnumericSpreadsheet");
2579                                         }
2580                                 }
2581                         }
2582                         if (macro)
2583                                 parse_macro(p, os, context);
2584                 }
2585
2586                 else if (t.cs() == "noindent") {
2587                         p.skip_spaces();
2588                         context.add_par_extra_stuff("\\noindent\n");
2589                 }
2590
2591                 else if (t.cs() == "appendix") {
2592                         context.add_par_extra_stuff("\\start_of_appendix\n");
2593                         // We need to start a new paragraph. Otherwise the
2594                         // appendix in 'bla\appendix\chapter{' would start
2595                         // too late.
2596                         context.new_paragraph(os);
2597                         // We need to make sure that the paragraph is
2598                         // generated even if it is empty. Otherwise the
2599                         // appendix in '\par\appendix\par\chapter{' would
2600                         // start too late.
2601                         context.check_layout(os);
2602                         // FIXME: This is a hack to prevent paragraph
2603                         // deletion if it is empty. Handle this better!
2604                         handle_comment(os,
2605                                 "%dummy comment inserted by tex2lyx to "
2606                                 "ensure that this paragraph is not empty",
2607                                 context);
2608                         // Both measures above may generate an additional
2609                         // empty paragraph, but that does not hurt, because
2610                         // whitespace does not matter here.
2611                         eat_whitespace(p, os, context, true);
2612                 }
2613
2614                 // Must catch empty dates before findLayout is called below
2615                 else if (t.cs() == "date") {
2616                         eat_whitespace(p, os, context, false);
2617                         p.pushPosition();
2618                         string const date = p.verbatim_item();
2619                         p.popPosition();
2620                         if (date.empty()) {
2621                                 preamble.suppressDate(true);
2622                                 p.verbatim_item();
2623                         } else {
2624                                 preamble.suppressDate(false);
2625                                 if (context.new_layout_allowed &&
2626                                     (newlayout = findLayout(context.textclass,
2627                                                             t.cs(), true))) {
2628                                         // write the layout
2629                                         output_command_layout(os, p, outer,
2630                                                         context, newlayout);
2631                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
2632                                         if (!preamble.titleLayoutFound())
2633                                                 preamble.titleLayoutFound(newlayout->intitle);
2634                                         set<string> const & req = newlayout->requires();
2635                                         set<string>::const_iterator it = req.begin();
2636                                         set<string>::const_iterator en = req.end();
2637                                         for (; it != en; ++it)
2638                                                 preamble.registerAutomaticallyLoadedPackage(*it);
2639                                 } else
2640                                         handle_ert(os,
2641                                                 "\\date{" + p.verbatim_item() + '}',
2642                                                 context);
2643                         }
2644                 }
2645
2646                 // Starred section headings
2647                 // Must attempt to parse "Section*" before "Section".
2648                 else if ((p.next_token().asInput() == "*") &&
2649                          context.new_layout_allowed &&
2650                          (newlayout = findLayout(context.textclass, t.cs() + '*', true))) {
2651                         // write the layout
2652                         p.get_token();
2653                         output_command_layout(os, p, outer, context, newlayout);
2654                         p.skip_spaces();
2655                         if (!preamble.titleLayoutFound())
2656                                 preamble.titleLayoutFound(newlayout->intitle);
2657                         set<string> const & req = newlayout->requires();
2658                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2659                                 preamble.registerAutomaticallyLoadedPackage(*it);
2660                 }
2661
2662                 // Section headings and the like
2663                 else if (context.new_layout_allowed &&
2664                          (newlayout = findLayout(context.textclass, t.cs(), true))) {
2665                         // write the layout
2666                         output_command_layout(os, p, outer, context, newlayout);
2667                         p.skip_spaces();
2668                         if (!preamble.titleLayoutFound())
2669                                 preamble.titleLayoutFound(newlayout->intitle);
2670                         set<string> const & req = newlayout->requires();
2671                         for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
2672                                 preamble.registerAutomaticallyLoadedPackage(*it);
2673                 }
2674
2675                 else if (t.cs() == "caption") {
2676                         p.skip_spaces();
2677                         context.check_layout(os);
2678                         p.skip_spaces();
2679                         begin_inset(os, "Caption\n");
2680                         Context newcontext(true, context.textclass);
2681                         newcontext.font = context.font;
2682                         newcontext.check_layout(os);
2683                         if (p.next_token().cat() != catEscape &&
2684                             p.next_token().character() == '[') {
2685                                 p.get_token(); // eat '['
2686                                 begin_inset(os, "Argument\n");
2687                                 os << "status collapsed\n";
2688                                 parse_text_in_inset(p, os, FLAG_BRACK_LAST, outer, context);
2689                                 end_inset(os);
2690                                 eat_whitespace(p, os, context, false);
2691                         }
2692                         parse_text(p, os, FLAG_ITEM, outer, context);
2693                         context.check_end_layout(os);
2694                         // We don't need really a new paragraph, but
2695                         // we must make sure that the next item gets a \begin_layout.
2696                         context.new_paragraph(os);
2697                         end_inset(os);
2698                         p.skip_spaces();
2699                         newcontext.check_end_layout(os);
2700                 }
2701
2702                 else if (t.cs() == "subfloat") {
2703                         // the syntax is \subfloat[caption]{content}
2704                         // if it is a table of figure depends on the surrounding float
2705                         bool has_caption = false;
2706                         p.skip_spaces();
2707                         // do nothing if there is no outer float
2708                         if (!float_type.empty()) {
2709                                 context.check_layout(os);
2710                                 p.skip_spaces();
2711                                 begin_inset(os, "Float " + float_type + "\n");
2712                                 os << "wide false"
2713                                    << "\nsideways false"
2714                                    << "\nstatus collapsed\n\n";
2715                                 // test for caption
2716                                 string caption;
2717                                 if (p.next_token().cat() != catEscape &&
2718                                                 p.next_token().character() == '[') {
2719                                                         p.get_token(); // eat '['
2720                                                         caption = parse_text_snippet(p, FLAG_BRACK_LAST, outer, context);
2721                                                         has_caption = true;
2722                                 }
2723                                 // the content
2724                                 parse_text_in_inset(p, os, FLAG_ITEM, outer, context);
2725                                 // the caption comes always as the last
2726                                 if (has_caption) {
2727                                         // we must make sure that the caption gets a \begin_layout
2728                                         os << "\n\\begin_layout Plain Layout";
2729                                         p.skip_spaces();
2730                                         begin_inset(os, "Caption\n");
2731                                         Context newcontext(true, context.textclass);
2732                                         newcontext.font = context.font;
2733                                         newcontext.check_layout(os);
2734                                         os << caption << "\n";
2735                                         newcontext.check_end_layout(os);
2736                                         // We don't need really a new paragraph, but
2737                                         // we must make sure that the next item gets a \begin_layout.
2738                                         //newcontext.new_paragraph(os);
2739                                         end_inset(os);
2740                                         p.skip_spaces();
2741                                 }
2742                                 // We don't need really a new paragraph, but
2743                                 // we must make sure that the next item gets a \begin_layout.
2744                                 if (has_caption)
2745                                         context.new_paragraph(os);
2746                                 end_inset(os);
2747                                 p.skip_spaces();
2748                                 context.check_end_layout(os);
2749                                 // close the layout we opened
2750                                 if (has_caption)
2751                                         os << "\n\\end_layout\n";
2752                         } else {
2753                                 // if the float type is not supported or there is no surrounding float
2754                                 // output it as ERT
2755                                 if (p.hasOpt()) {
2756                                         string opt_arg = convert_command_inset_arg(p.getArg('[', ']'));
2757                                         handle_ert(os, t.asInput() + '[' + opt_arg +
2758                                                "]{" + p.verbatim_item() + '}', context);
2759                                 } else
2760                                         handle_ert(os, t.asInput() + "{" + p.verbatim_item() + '}', context);
2761                         }
2762                 }
2763
2764                 else if (t.cs() == "includegraphics") {
2765                         bool const clip = p.next_token().asInput() == "*";
2766                         if (clip)
2767                                 p.get_token();
2768                         string const arg = p.getArg('[', ']');
2769                         map<string, string> opts;
2770                         vector<string> keys;
2771                         split_map(arg, opts, keys);
2772                         if (clip)
2773                                 opts["clip"] = string();
2774                         string name = normalize_filename(p.verbatim_item());
2775
2776                         string const path = getMasterFilePath();
2777                         // We want to preserve relative / absolute filenames,
2778                         // therefore path is only used for testing
2779                         if (!makeAbsPath(name, path).exists()) {
2780                                 // The file extension is probably missing.
2781                                 // Now try to find it out.
2782                                 string const dvips_name =
2783                                         find_file(name, path,
2784                                                   known_dvips_graphics_formats);
2785                                 string const pdftex_name =
2786                                         find_file(name, path,
2787                                                   known_pdftex_graphics_formats);
2788                                 if (!dvips_name.empty()) {
2789                                         if (!pdftex_name.empty()) {
2790                                                 cerr << "This file contains the "
2791                                                         "latex snippet\n"
2792                                                         "\"\\includegraphics{"
2793                                                      << name << "}\".\n"
2794                                                         "However, files\n\""
2795                                                      << dvips_name << "\" and\n\""
2796                                                      << pdftex_name << "\"\n"
2797                                                         "both exist, so I had to make a "
2798                                                         "choice and took the first one.\n"
2799                                                         "Please move the unwanted one "
2800                                                         "someplace else and try again\n"
2801                                                         "if my choice was wrong."
2802                                                      << endl;
2803                                         }
2804                                         name = dvips_name;
2805                                 } else if (!pdftex_name.empty()) {
2806                                         name = pdftex_name;
2807                                         pdflatex = true;
2808                                 }
2809                         }
2810
2811                         if (makeAbsPath(name, path).exists())
2812                                 fix_relative_filename(name);
2813                         else
2814                                 cerr << "Warning: Could not find graphics file '"
2815                                      << name << "'." << endl;
2816
2817                         context.check_layout(os);
2818                         begin_inset(os, "Graphics ");
2819                         os << "\n\tfilename " << name << '\n';
2820                         if (opts.find("width") != opts.end())
2821                                 os << "\twidth "
2822                                    << translate_len(opts["width"]) << '\n';
2823                         if (opts.find("height") != opts.end())
2824                                 os << "\theight "
2825                                    << translate_len(opts["height"]) << '\n';
2826                         if (opts.find("scale") != opts.end()) {
2827                                 istringstream iss(opts["scale"]);
2828                                 double val;
2829                                 iss >> val;
2830                                 val = val*100;
2831                                 os << "\tscale " << val << '\n';
2832                         }
2833                         if (opts.find("angle") != opts.end()) {
2834                                 os << "\trotateAngle "
2835                                    << opts["angle"] << '\n';
2836                                 vector<string>::const_iterator a =
2837                                         find(keys.begin(), keys.end(), "angle");
2838                                 vector<string>::const_iterator s =
2839                                         find(keys.begin(), keys.end(), "width");
2840                                 if (s == keys.end())
2841                                         s = find(keys.begin(), keys.end(), "height");
2842                                 if (s == keys.end())
2843                                         s = find(keys.begin(), keys.end(), "scale");
2844                                 if (s != keys.end() && distance(s, a) > 0)
2845                                         os << "\tscaleBeforeRotation\n";
2846                         }
2847                         if (opts.find("origin") != opts.end()) {
2848                                 ostringstream ss;
2849                                 string const opt = opts["origin"];
2850                                 if (opt.find('l') != string::npos) ss << "left";
2851                                 if (opt.find('r') != string::npos) ss << "right";
2852                                 if (opt.find('c') != string::npos) ss << "center";
2853                                 if (opt.find('t') != string::npos) ss << "Top";
2854                                 if (opt.find('b') != string::npos) ss << "Bottom";
2855                                 if (opt.find('B') != string::npos) ss << "Baseline";
2856                                 if (!ss.str().empty())
2857                                         os << "\trotateOrigin " << ss.str() << '\n';
2858                                 else
2859                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
2860                         }
2861                         if (opts.find("keepaspectratio") != opts.end())
2862                                 os << "\tkeepAspectRatio\n";
2863                         if (opts.find("clip") != opts.end())
2864                                 os << "\tclip\n";
2865                         if (opts.find("draft") != opts.end())
2866                                 os << "\tdraft\n";
2867                         if (opts.find("bb") != opts.end())
2868                                 os << "\tBoundingBox "
2869                                    << opts["bb"] << '\n';
2870                         int numberOfbbOptions = 0;
2871                         if (opts.find("bbllx") != opts.end())
2872                                 numberOfbbOptions++;
2873                         if (opts.find("bblly") != opts.end())
2874                                 numberOfbbOptions++;
2875                         if (opts.find("bburx") != opts.end())
2876                                 numberOfbbOptions++;
2877                         if (opts.find("bbury") != opts.end())
2878                                 numberOfbbOptions++;
2879                         if (numberOfbbOptions == 4)
2880                                 os << "\tBoundingBox "
2881                                    << opts["bbllx"] << " " << opts["bblly"] << " "
2882                                    << opts["bburx"] << " " << opts["bbury"] << '\n';
2883                         else if (numberOfbbOptions > 0)
2884                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2885                         numberOfbbOptions = 0;
2886                         if (opts.find("natwidth") != opts.end())
2887                                 numberOfbbOptions++;
2888                         if (opts.find("natheight") != opts.end())
2889                                 numberOfbbOptions++;
2890                         if (numberOfbbOptions == 2)
2891                                 os << "\tBoundingBox 0bp 0bp "
2892                                    << opts["natwidth"] << " " << opts["natheight"] << '\n';
2893                         else if (numberOfbbOptions > 0)
2894                                 cerr << "Warning: Ignoring incomplete includegraphics boundingbox arguments.\n";
2895                         ostringstream special;
2896                         if (opts.find("hiresbb") != opts.end())
2897                                 special << "hiresbb,";
2898                         if (opts.find("trim") != opts.end())
2899                                 special << "trim,";
2900                         if (opts.find("viewport") != opts.end())
2901                                 special << "viewport=" << opts["viewport"] << ',';
2902                         if (opts.find("totalheight") != opts.end())
2903                                 special << "totalheight=" << opts["totalheight"] << ',';
2904                         if (opts.find("type") != opts.end())
2905                                 special << "type=" << opts["type"] << ',';
2906                         if (opts.find("ext") != opts.end())
2907                                 special << "ext=" << opts["ext"] << ',';
2908                         if (opts.find("read") != opts.end())
2909                                 special << "read=" << opts["read"] << ',';
2910                         if (opts.find("command") != opts.end())
2911                                 special << "command=" << opts["command"] << ',';
2912                         string s_special = special.str();
2913                         if (!s_special.empty()) {
2914                                 // We had special arguments. Remove the trailing ','.
2915                                 os << "\tspecial " << s_special.substr(0, s_special.size() - 1) << '\n';
2916                         }
2917                         // TODO: Handle the unknown settings better.
2918                         // Warn about invalid options.
2919                         // Check whether some option was given twice.
2920                         end_inset(os);
2921                         preamble.registerAutomaticallyLoadedPackage("graphicx");
2922                 }
2923
2924                 else if (t.cs() == "footnote" ||
2925                          (t.cs() == "thanks" && context.layout->intitle)) {
2926                         p.skip_spaces();
2927                         context.check_layout(os);
2928                         begin_inset(os, "Foot\n");
2929                         os << "status collapsed\n\n";
2930                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2931                         end_inset(os);
2932                 }
2933
2934                 else if (t.cs() == "marginpar") {
2935                         p.skip_spaces();
2936                         context.check_layout(os);
2937                         begin_inset(os, "Marginal\n");
2938                         os << "status collapsed\n\n";
2939                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
2940                         end_inset(os);
2941                 }
2942
2943                 else if (t.cs() == "lstinline") {
2944                         p.skip_spaces();
2945                         parse_listings(p, os, context, true);
2946                 }
2947
2948                 else if (t.cs() == "ensuremath") {
2949                         p.skip_spaces();
2950                         context.check_layout(os);
2951                         string const s = p.verbatim_item();
2952                         //FIXME: this never triggers in UTF8
2953                         if (s == "\xb1" || s == "\xb3" || s == "\xb2" || s == "\xb5")
2954                                 os << s;
2955                         else
2956                                 handle_ert(os, "\\ensuremath{" + s + "}",
2957                                            context);
2958                 }
2959
2960                 else if (t.cs() == "makeindex" || t.cs() == "maketitle") {
2961                         if (preamble.titleLayoutFound()) {
2962                                 // swallow this
2963                                 skip_spaces_braces(p);
2964                         } else
2965                                 handle_ert(os, t.asInput(), context);
2966                 }
2967
2968                 else if (t.cs() == "tableofcontents" || t.cs() == "lstlistoflistings") {
2969                         context.check_layout(os);
2970                         begin_command_inset(os, "toc", t.cs());
2971                         end_inset(os);
2972                         skip_spaces_braces(p);
2973                         if (t.cs() == "lstlistoflistings")
2974                                 preamble.registerAutomaticallyLoadedPackage("listings");
2975                 }
2976
2977                 else if (t.cs() == "listoffigures") {
2978                         context.check_layout(os);
2979                         begin_inset(os, "FloatList figure\n");
2980                         end_inset(os);
2981                         skip_spaces_braces(p);
2982                 }
2983
2984                 else if (t.cs() == "listoftables") {
2985                         context.check_layout(os);
2986                         begin_inset(os, "FloatList table\n");
2987                         end_inset(os);
2988                         skip_spaces_braces(p);
2989                 }
2990
2991                 else if (t.cs() == "listof") {
2992                         p.skip_spaces(true);
2993                         string const name = p.get_token().cs();
2994                         if (context.textclass.floats().typeExist(name)) {
2995                                 context.check_layout(os);
2996                                 begin_inset(os, "FloatList ");
2997                                 os << name << "\n";
2998                                 end_inset(os);
2999                                 p.get_token(); // swallow second arg
3000                         } else
3001                                 handle_ert(os, "\\listof{" + name + "}", context);
3002                 }
3003
3004                 else if ((where = is_known(t.cs(), known_text_font_families)))
3005                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3006                                 context, "\\family", context.font.family,
3007                                 known_coded_font_families[where - known_text_font_families]);
3008
3009                 else if ((where = is_known(t.cs(), known_text_font_series)))
3010                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3011                                 context, "\\series", context.font.series,
3012                                 known_coded_font_series[where - known_text_font_series]);
3013
3014                 else if ((where = is_known(t.cs(), known_text_font_shapes)))
3015                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3016                                 context, "\\shape", context.font.shape,
3017                                 known_coded_font_shapes[where - known_text_font_shapes]);
3018
3019                 else if (t.cs() == "textnormal" || t.cs() == "normalfont") {
3020                         context.check_layout(os);
3021                         TeXFont oldFont = context.font;
3022                         context.font.init();
3023                         context.font.size = oldFont.size;
3024                         os << "\n\\family " << context.font.family << "\n";
3025                         os << "\n\\series " << context.font.series << "\n";
3026                         os << "\n\\shape " << context.font.shape << "\n";
3027                         if (t.cs() == "textnormal") {
3028                                 parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3029                                 output_font_change(os, context.font, oldFont);
3030                                 context.font = oldFont;
3031                         } else
3032                                 eat_whitespace(p, os, context, false);
3033                 }
3034
3035                 else if (t.cs() == "textcolor") {
3036                         // scheme is \textcolor{color name}{text}
3037                         string const color = p.verbatim_item();
3038                         // we only support the predefined colors of the color package
3039                         if (color == "black" || color == "blue" || color == "cyan"
3040                                 || color == "green" || color == "magenta" || color == "red"
3041                                 || color == "white" || color == "yellow") {
3042                                         context.check_layout(os);
3043                                         os << "\n\\color " << color << "\n";
3044                                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3045                                         context.check_layout(os);
3046                                         os << "\n\\color inherit\n";
3047                                         preamble.registerAutomaticallyLoadedPackage("color");
3048                         } else
3049                                 // for custom defined colors
3050                                 handle_ert(os, t.asInput() + "{" + color + "}", context);
3051                 }
3052
3053                 else if (t.cs() == "underbar" || t.cs() == "uline") {
3054                         // \underbar is not 100% correct (LyX outputs \uline
3055                         // of ulem.sty). The difference is that \ulem allows
3056                         // line breaks, and \underbar does not.
3057                         // Do NOT handle \underline.
3058                         // \underbar cuts through y, g, q, p etc.,
3059                         // \underline does not.
3060                         context.check_layout(os);
3061                         os << "\n\\bar under\n";
3062                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3063                         context.check_layout(os);
3064                         os << "\n\\bar default\n";
3065                         preamble.registerAutomaticallyLoadedPackage("ulem");
3066                 }
3067
3068                 else if (t.cs() == "sout") {
3069                         context.check_layout(os);
3070                         os << "\n\\strikeout on\n";
3071                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3072                         context.check_layout(os);
3073                         os << "\n\\strikeout default\n";
3074                         preamble.registerAutomaticallyLoadedPackage("ulem");
3075                 }
3076
3077                 else if (t.cs() == "uuline" || t.cs() == "uwave" ||
3078                          t.cs() == "emph" || t.cs() == "noun") {
3079                         context.check_layout(os);
3080                         os << "\n\\" << t.cs() << " on\n";
3081                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3082                         context.check_layout(os);
3083                         os << "\n\\" << t.cs() << " default\n";
3084                         if (t.cs() == "uuline" || t.cs() == "uwave")
3085                                 preamble.registerAutomaticallyLoadedPackage("ulem");
3086                 }
3087
3088                 else if (t.cs() == "lyxadded" || t.cs() == "lyxdeleted") {
3089                         context.check_layout(os);
3090                         string name = p.getArg('{', '}');
3091                         string localtime = p.getArg('{', '}');
3092                         preamble.registerAuthor(name);
3093                         Author const & author = preamble.getAuthor(name);
3094                         // from_ctime() will fail if LyX decides to output the
3095                         // time in the text language. It might also use a wrong
3096                         // time zone (if the original LyX document was exported
3097                         // with a different time zone).
3098                         time_t ptime = from_ctime(localtime);
3099                         if (ptime == static_cast<time_t>(-1)) {
3100                                 cerr << "Warning: Could not parse time `" << localtime
3101                                      << "´ for change tracking, using current time instead.\n";
3102                                 ptime = current_time();
3103                         }
3104                         if (t.cs() == "lyxadded")
3105                                 os << "\n\\change_inserted ";
3106                         else
3107                                 os << "\n\\change_deleted ";
3108                         os << author.bufferId() << ' ' << ptime << '\n';
3109                         parse_text_snippet(p, os, FLAG_ITEM, outer, context);
3110                         bool dvipost    = LaTeXPackages::isAvailable("dvipost");
3111                         bool xcolorulem = LaTeXPackages::isAvailable("ulem") &&
3112                                           LaTeXPackages::isAvailable("xcolor");
3113                         // No need to test for luatex, since luatex comes in
3114                         // two flavours (dvi and pdf), like latex, and those
3115                         // are detected by pdflatex.
3116                         if (pdflatex || xetex) {
3117                                 if (xcolorulem) {
3118                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3119                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3120                                         preamble.registerAutomaticallyLoadedPackage("pdfcolmk");
3121                                 }
3122                         } else {
3123                                 if (dvipost) {
3124                                         preamble.registerAutomaticallyLoadedPackage("dvipost");
3125                                 } else if (xcolorulem) {
3126                                         preamble.registerAutomaticallyLoadedPackage("ulem");
3127                                         preamble.registerAutomaticallyLoadedPackage("xcolor");
3128                                 }
3129                         }
3130                 }
3131
3132                 else if (t.cs() == "phantom" || t.cs() == "hphantom" ||
3133                              t.cs() == "vphantom") {
3134                         context.check_layout(os);
3135                         if (t.cs() == "phantom")
3136                                 begin_inset(os, "Phantom Phantom\n");
3137                         if (t.cs() == "hphantom")
3138                                 begin_inset(os, "Phantom HPhantom\n");
3139                         if (t.cs() == "vphantom")
3140                                 begin_inset(os, "Phantom VPhantom\n");
3141                         os << "status open\n";
3142                         parse_text_in_inset(p, os, FLAG_ITEM, outer, context,
3143                                             "Phantom");
3144                         end_inset(os);
3145                 }
3146
3147                 else if (t.cs() == "href") {
3148                         context.check_layout(os);
3149                         string target = p.getArg('{', '}');
3150                         string name = p.getArg('{', '}');
3151                         string type;
3152                         size_t i = target.find(':');
3153                         if (i != string::npos) {
3154                                 type = target.substr(0, i + 1);
3155                                 if (type == "mailto:" || type == "file:")
3156                                         target = target.substr(i + 1);
3157                                 // handle the case that name is equal to target, except of "http://"
3158                                 else if (target.substr(i + 3) == name && type == "http:")
3159                                         target = name;
3160                         }
3161                         begin_command_inset(os, "href", "href");
3162                         if (name != target)
3163                                 os << "name \"" << name << "\"\n";
3164                         os << "target \"" << target << "\"\n";
3165                         if (type == "mailto:" || type == "file:")
3166                                 os << "type \"" << type << "\"\n";
3167                         end_inset(os);
3168                         skip_spaces_braces(p);
3169                 }
3170
3171                 else if (t.cs() == "lyxline") {
3172                         // swallow size argument (it is not used anyway)
3173                         p.getArg('{', '}');
3174                         if (!context.atParagraphStart()) {
3175                                 // so our line is in the middle of a paragraph
3176                                 // we need to add a new line, lest this line
3177                                 // follow the other content on that line and
3178                                 // run off the side of the page
3179                                 // FIXME: This may create an empty paragraph,
3180                                 //        but without that it would not be
3181                                 //        possible to set noindent below.
3182                                 //        Fortunately LaTeX does not care
3183                                 //        about the empty paragraph.
3184                                 context.new_paragraph(os);
3185                         }
3186                         if (preamble.indentParagraphs()) {
3187                                 // we need to unindent, lest the line be too long
3188                                 context.add_par_extra_stuff("\\noindent\n");
3189                         }
3190                         context.check_layout(os);
3191                         begin_command_inset(os, "line", "rule");
3192                         os << "offset \"0.5ex\"\n"
3193                               "width \"100line%\"\n"
3194                               "height \"1pt\"\n";
3195                         end_inset(os);
3196                 }
3197
3198                 else if (t.cs() == "rule") {
3199                         string const offset = (p.hasOpt() ? p.getArg('[', ']') : string());
3200                         string const width = p.getArg('{', '}');
3201                         string const thickness = p.getArg('{', '}');
3202                         context.check_layout(os);
3203                         begin_command_inset(os, "line", "rule");
3204                         if (!offset.empty())
3205                                 os << "offset \"" << translate_len(offset) << "\"\n";
3206                         os << "width \"" << translate_len(width) << "\"\n"
3207                                   "height \"" << translate_len(thickness) << "\"\n";
3208                         end_inset(os);
3209                 }
3210
3211                 else if (is_known(t.cs(), known_phrases) ||
3212                          (t.cs() == "protect" &&
3213                           p.next_token().cat() == catEscape &&
3214                           is_known(p.next_token().cs(), known_phrases))) {
3215                         // LyX sometimes puts a \protect in front, so we have to ignore it
3216                         // FIXME: This needs to be changed when bug 4752 is fixed.
3217                         where = is_known(
3218                                 t.cs() == "protect" ? p.get_token().cs() : t.cs(),
3219                                 known_phrases);
3220                         context.check_layout(os);
3221                         os << known_coded_phrases[where - known_phrases];
3222                         skip_spaces_braces(p);
3223                 }
3224
3225                 else if ((where = is_known(t.cs(), known_ref_commands))) {
3226                         string const opt = p.getOpt();
3227                         if (opt.empty()) {
3228                                 context.check_layout(os);
3229                                 begin_command_inset(os, "ref",
3230                                         known_coded_ref_commands[where - known_ref_commands]);
3231                                 os << "reference \""
3232                                    << convert_command_inset_arg(p.verbatim_item())
3233                                    << "\"\n";
3234                                 end_inset(os);
3235                                 if (t.cs() == "vref" || t.cs() == "vpageref")
3236                                         preamble.registerAutomaticallyLoadedPackage("varioref");
3237
3238                         } else {
3239                                 // LyX does not support optional arguments of ref commands
3240                                 handle_ert(os, t.asInput() + '[' + opt + "]{" +
3241                                                p.verbatim_item() + "}", context);
3242                         }
3243                 }
3244
3245                 else if (use_natbib &&
3246                          is_known(t.cs(), known_natbib_commands) &&
3247                          ((t.cs() != "citefullauthor" &&
3248                            t.cs() != "citeyear" &&
3249                            t.cs() != "citeyearpar") ||
3250                           p.next_token().asInput() != "*")) {
3251                         context.check_layout(os);
3252                         string command = t.cs();
3253                         if (p.next_token().asInput() == "*") {
3254                                 command += '*';
3255                                 p.get_token();
3256                         }
3257                         if (command == "citefullauthor")
3258                                 // alternative name for "\\citeauthor*"
3259                                 command = "citeauthor*";
3260
3261                         // text before the citation
3262                         string before;
3263                         // text after the citation
3264                         string after;
3265                         get_cite_arguments(p, true, before, after);
3266
3267                         if (command == "cite") {
3268                                 // \cite without optional argument means
3269                                 // \citet, \cite with at least one optional
3270                                 // argument means \citep.
3271                                 if (before.empty() && after.empty())
3272                                         command = "citet";
3273                                 else
3274                                         command = "citep";
3275                         }
3276                         if (before.empty() && after == "[]")
3277                                 // avoid \citet[]{a}
3278                                 after.erase();
3279                         else if (before == "[]" && after == "[]") {
3280                                 // avoid \citet[][]{a}
3281                                 before.erase();
3282                                 after.erase();
3283                         }
3284                         // remove the brackets around after and before
3285                         if (!after.empty()) {
3286                                 after.erase(0, 1);
3287                                 after.erase(after.length() - 1, 1);
3288                                 after = convert_command_inset_arg(after);
3289                         }
3290                         if (!before.empty()) {
3291                                 before.erase(0, 1);
3292                                 before.erase(before.length() - 1, 1);
3293                                 before = convert_command_inset_arg(before);
3294                         }
3295                         begin_command_inset(os, "citation", command);
3296                         os << "after " << '"' << after << '"' << "\n";
3297                         os << "before " << '"' << before << '"' << "\n";
3298                         os << "key \""
3299                            << convert_command_inset_arg(p.verbatim_item())
3300                            << "\"\n";
3301                         end_inset(os);
3302                 }
3303
3304                 else if (use_jurabib &&
3305                          is_known(t.cs(), known_jurabib_commands) &&
3306                          (t.cs() == "cite" || p.next_token().asInput() != "*")) {
3307                         context.check_layout(os);
3308                         string command = t.cs();
3309                         if (p.next_token().asInput() == "*") {
3310                                 command += '*';
3311                                 p.get_token();
3312                         }
3313                         char argumentOrder = '\0';
3314                         vector<string> const options =
3315                                 preamble.getPackageOptions("jurabib");
3316                         if (find(options.begin(), options.end(),
3317                                       "natbiborder") != options.end())
3318                                 argumentOrder = 'n';
3319                         else if (find(options.begin(), options.end(),
3320                                            "jurabiborder") != options.end())
3321                                 argumentOrder = 'j';
3322
3323                         // text before the citation
3324                         string before;
3325                         // text after the citation
3326                         string after;
3327                         get_cite_arguments(p, argumentOrder != 'j', before, after);
3328
3329                         string const citation = p.verbatim_item();
3330                         if (!before.empty() && argumentOrder == '\0') {
3331                                 cerr << "Warning: Assuming argument order "
3332                                         "of jurabib version 0.6 for\n'"
3333                                      << command << before << after << '{'
3334                                      << citation << "}'.\n"
3335                                         "Add 'jurabiborder' to the jurabib "
3336                                         "package options if you used an\n"
3337                                         "earlier jurabib version." << endl;
3338                         }
3339                         if (!after.empty()) {
3340                                 after.erase(0, 1);
3341                                 after.erase(after.length() - 1, 1);
3342                         }
3343                         if (!before.empty()) {
3344                                 before.erase(0, 1);
3345                                 before.erase(before.length() - 1, 1);
3346                         }
3347                         begin_command_inset(os, "citation", command);
3348                         os << "after " << '"' << after << '"' << "\n";
3349                         os << "before " << '"' << before << '"' << "\n";
3350                         os << "key " << '"' << citation << '"' << "\n";
3351                         end_inset(os);
3352                 }
3353
3354                 else if (t.cs() == "cite"
3355                         || t.cs() == "nocite") {
3356                         context.check_layout(os);
3357                         string after = convert_command_inset_arg(p.getArg('[', ']'));
3358                         string key = convert_command_inset_arg(p.verbatim_item());
3359                         // store the case that it is "\nocite{*}" to use it later for
3360                         // the BibTeX inset
3361                         if (key != "*") {
3362                                 begin_command_inset(os, "citation", t.cs());
3363                                 os << "after " << '"' << after << '"' << "\n";
3364                                 os << "key " << '"' << key << '"' << "\n";
3365                                 end_inset(os);
3366                         } else if (t.cs() == "nocite")
3367                                 btprint = key;
3368                 }
3369
3370                 else if (t.cs() == "index" ||
3371                          (t.cs() == "sindex" && preamble.use_indices() == "true")) {
3372                         context.check_layout(os);
3373                         string const arg = (t.cs() == "sindex" && p.hasOpt()) ?
3374                                 p.getArg('[', ']') : "";
3375                         string const kind = arg.empty() ? "idx" : arg;
3376                         begin_inset(os, "Index ");
3377                         os << kind << "\nstatus collapsed\n";
3378                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, "Index");
3379                         end_inset(os);
3380                         if (kind != "idx")
3381                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3382                 }
3383
3384                 else if (t.cs() == "nomenclature") {
3385                         context.check_layout(os);
3386                         begin_command_inset(os, "nomenclature", "nomenclature");
3387                         string prefix = convert_command_inset_arg(p.getArg('[', ']'));
3388                         if (!prefix.empty())
3389                                 os << "prefix " << '"' << prefix << '"' << "\n";
3390                         os << "symbol " << '"'
3391                            << convert_command_inset_arg(p.verbatim_item());
3392                         os << "\"\ndescription \""
3393                            << convert_command_inset_arg(p.verbatim_item())
3394                            << "\"\n";
3395                         end_inset(os);
3396                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3397                 }
3398
3399                 else if (t.cs() == "label") {
3400                         context.check_layout(os);
3401                         begin_command_inset(os, "label", "label");
3402                         os << "name \""
3403                            << convert_command_inset_arg(p.verbatim_item())
3404                            << "\"\n";
3405                         end_inset(os);
3406                 }
3407
3408                 else if (t.cs() == "printindex") {
3409                         context.check_layout(os);
3410                         begin_command_inset(os, "index_print", "printindex");
3411                         os << "type \"idx\"\n";
3412                         end_inset(os);
3413                         skip_spaces_braces(p);
3414                         preamble.registerAutomaticallyLoadedPackage("makeidx");
3415                         if (preamble.use_indices() == "true")
3416                                 preamble.registerAutomaticallyLoadedPackage("splitidx");
3417                 }
3418
3419                 else if (t.cs() == "printnomenclature") {
3420                         string width = "";
3421                         string width_type = "";
3422                         context.check_layout(os);
3423                         begin_command_inset(os, "nomencl_print", "printnomenclature");
3424                         // case of a custom width
3425                         if (p.hasOpt()) {
3426                                 width = p.getArg('[', ']');
3427                                 width = translate_len(width);
3428                                 width_type = "custom";
3429                         }
3430                         // case of no custom width
3431                         // the case of no custom width but the width set
3432                         // via \settowidth{\nomlabelwidth}{***} cannot be supported
3433                         // because the user could have set anything, not only the width
3434                         // of the longest label (which would be width_type = "auto")
3435                         string label = convert_command_inset_arg(p.getArg('{', '}'));
3436                         if (label.empty() && width_type.empty())
3437                                 width_type = "none";
3438                         os << "set_width \"" << width_type << "\"\n";
3439                         if (width_type == "custom")
3440                                 os << "width \"" << width << '\"';
3441                         end_inset(os);
3442                         skip_spaces_braces(p);
3443                         preamble.registerAutomaticallyLoadedPackage("nomencl");
3444                 }
3445
3446                 else if ((t.cs() == "textsuperscript" || t.cs() == "textsubscript")) {
3447                         context.check_layout(os);
3448                         begin_inset(os, "script ");
3449                         os << t.cs().substr(4) << '\n';
3450                         parse_text_in_inset(p, os, FLAG_ITEM, false, context);
3451                         end_inset(os);
3452                         if (t.cs() == "textsubscript")
3453                                 preamble.registerAutomaticallyLoadedPackage("subscript");
3454                 }
3455
3456                 else if ((where = is_known(t.cs(), known_quotes))) {
3457                         context.check_layout(os);
3458                         begin_inset(os, "Quotes ");
3459                         os << known_coded_quotes[where - known_quotes];
3460                         end_inset(os);
3461                         // LyX adds {} after the quote, so we have to eat
3462                         // spaces here if there are any before a possible
3463                         // {} pair.
3464                         eat_whitespace(p, os, context, false);
3465                         skip_braces(p);
3466                 }
3467
3468                 else if ((where = is_known(t.cs(), known_sizes)) &&
3469                          context.new_layout_allowed) {
3470                         context.check_layout(os);
3471                         TeXFont const oldFont = context.font;
3472                         context.font.size = known_coded_sizes[where - known_sizes];
3473                         output_font_change(os, oldFont, context.font);
3474                         eat_whitespace(p, os, context, false);
3475                 }
3476
3477                 else if ((where = is_known(t.cs(), known_font_families)) &&
3478                          context.new_layout_allowed) {
3479                         context.check_layout(os);
3480                         TeXFont const oldFont = context.font;
3481                         context.font.family =
3482                                 known_coded_font_families[where - known_font_families];
3483                         output_font_change(os, oldFont, context.font);
3484                         eat_whitespace(p, os, context, false);
3485                 }
3486
3487                 else if ((where = is_known(t.cs(), known_font_series)) &&
3488                          context.new_layout_allowed) {
3489                         context.check_layout(os);
3490                         TeXFont const oldFont = context.font;
3491                         context.font.series =
3492                                 known_coded_font_series[where - known_font_series];
3493                         output_font_change(os, oldFont, context.font);
3494                         eat_whitespace(p, os, context, false);
3495                 }
3496
3497                 else if ((where = is_known(t.cs(), known_font_shapes)) &&
3498                          context.new_layout_allowed) {
3499                         context.check_layout(os);
3500                         TeXFont const oldFont = context.font;
3501                         context.font.shape =
3502                                 known_coded_font_shapes[where - known_font_shapes];
3503                         output_font_change(os, oldFont, context.font);
3504                         eat_whitespace(p, os, context, false);
3505                 }
3506                 else if ((where = is_known(t.cs(), known_old_font_families)) &&
3507                          context.new_layout_allowed) {
3508                         context.check_layout(os);
3509                         TeXFont const oldFont = context.font;
3510                         context.font.init();
3511                         context.font.size = oldFont.size;
3512                         context.font.family =
3513                                 known_coded_font_families[where - known_old_font_families];
3514                         output_font_change(os, oldFont, context.font);
3515                         eat_whitespace(p, os, context, false);
3516                 }
3517
3518                 else if ((where = is_known(t.cs(), known_old_font_series)) &&
3519                          context.new_layout_allowed) {
3520                         context.check_layout(os);
3521                         TeXFont const oldFont = context.font;
3522                         context.font.init();
3523                         context.font.size = oldFont.size;
3524                         context.font.series =
3525                                 known_coded_font_series[where - known_old_font_series];
3526                         output_font_change(os, oldFont, context.font);
3527                         eat_whitespace(p, os, context, false);
3528                 }
3529
3530                 else if ((where = is_known(t.cs(), known_old_font_shapes)) &&
3531                          context.new_layout_allowed) {
3532                         context.check_layout(os);
3533                         TeXFont const oldFont = context.font;
3534                         context.font.init();
3535                         context.font.size = oldFont.size;
3536                         context.font.shape =
3537                                 known_coded_font_shapes[where - known_old_font_shapes];
3538                         output_font_change(os, oldFont, context.font);
3539                         eat_whitespace(p, os, context, false);
3540                 }
3541
3542                 else if (t.cs() == "selectlanguage") {
3543                         context.check_layout(os);
3544                         // save the language for the case that a
3545                         // \foreignlanguage is used
3546                         context.font.language = babel2lyx(p.verbatim_item());
3547                         os << "\n\\lang " << context.font.language << "\n";
3548                 }
3549
3550                 else if (t.cs() == "foreignlanguage") {
3551                         string const lang = babel2lyx(p.verbatim_item());
3552                         parse_text_attributes(p, os, FLAG_ITEM, outer,
3553                                               context, "\\lang",
3554                                               context.font.language, lang);
3555                 }
3556
3557                 else if (is_known(t.cs().substr(4, string::npos), polyglossia_languages)) {
3558                         // scheme is \textLANGUAGE{text} where LANGUAGE is in polyglossia_languages[]
3559                         string lang;
3560                         // We have to output the whole command if it has an option
3561                         // because LyX doesn't support this yet, see bug #8214,
3562                         // only if there is a single option specifying a variant, we can handle it.
3563                         if (p.hasOpt()) {
3564                                 string langopts = p.getOpt();
3565                                 // check if the option contains a variant, if yes, extract it
3566                                 string::size_type pos_var = langopts.find("variant");
3567                                 string::size_type i = langopts.find(',');
3568                                 if (pos_var != string::npos){
3569                                         string variant;
3570                                         if (i == string::npos) {
3571                                                 variant = langopts.substr(pos_var + 8, langopts.length() - pos_var - 9);
3572                                                 lang = polyglossia2lyx(variant);
3573                                                 parse_text_attributes(p, os, FLAG_ITEM, outer,
3574                                                                           context, "\\lang",
3575                                                                           context.font.language, lang);
3576                                         }
3577                                         else
3578                                                 handle_ert(os, t.asInput() + langopts, context);
3579                                 } else
3580                                         handle_ert(os, t.asInput() + langopts, context);
3581                         } else {
3582                                 lang = polyglossia2lyx(t.cs().substr(4, string::npos));
3583                                 parse_text_attributes(p, os, FLAG_ITEM, outer,
3584                                                           context, "\\lang",
3585                                                           context.font.language, lang);
3586                         }
3587                 }
3588
3589                 else if (t.cs() == "inputencoding") {
3590                         // nothing to write here
3591                         string const enc = subst(p.verbatim_item(), "\n", " ");
3592                         p.setEncoding(enc);
3593                 }
3594
3595                 else if ((where = is_known(t.cs(), known_special_chars))) {
3596                         context.check_layout(os);
3597                         os << "\\SpecialChar \\"
3598                            << known_coded_special_chars[where - known_special_chars]
3599                            << '\n';
3600                         skip_spaces_braces(p);
3601                 }
3602
3603                 else if (t.cs() == "nobreakdash" && p.next_token().asInput() == "-") {
3604                         context.check_layout(os);
3605                         os << "\\SpecialChar \\nobreakdash-\n";
3606                         p.get_token();
3607                 }
3608
3609                 else if (t.cs() == "textquotedbl") {
3610                         context.check_layout(os);
3611                         os << "\"";
3612                         skip_braces(p);
3613                 }
3614
3615                 else if (t.cs() == "@" && p.next_token().asInput() == ".") {
3616                         context.check_layout(os);
3617                         os << "\\SpecialChar \\@.\n";
3618                         p.get_token();
3619                 }
3620
3621                 else if (t.cs() == "-") {
3622                         context.check_layout(os);
3623                         os << "\\SpecialChar \\-\n";
3624                 }
3625
3626                 else if (t.cs() == "textasciitilde") {
3627                         context.check_layout(os);
3628                         os << '~';
3629                         skip_spaces_braces(p);
3630                 }
3631
3632                 else if (t.cs() == "textasciicircum") {
3633                         context.check_layout(os);
3634                         os << '^';
3635                         skip_spaces_braces(p);
3636                 }
3637
3638                 else if (t.cs() == "textbackslash") {
3639                         context.check_layout(os);
3640                         os << "\n\\backslash\n";
3641                         skip_spaces_braces(p);
3642                 }
3643
3644                 else if (t.cs() == "_" || t.cs() == "&" || t.cs() == "#"
3645                             || t.cs() == "$" || t.cs() == "{" || t.cs() == "}"
3646                             || t.cs() == "%") {
3647                         context.check_layout(os);
3648                         os << t.cs();
3649                 }
3650
3651                 else if (t.cs() == "char") {
3652                         context.check_layout(os);
3653                         if (p.next_token().character() == '`') {
3654                                 p.get_token();
3655                                 if (p.next_token().cs() == "\"") {
3656                                         p.get_token();
3657                                         os << '"';
3658                                         skip_braces(p);
3659                                 } else {
3660                                         handle_ert(os, "\\char`", context);
3661                                 }
3662                         } else {
3663                                 handle_ert(os, "\\char", context);
3664                         }
3665                 }
3666
3667                 else if (t.cs() == "verb") {
3668                         context.check_layout(os);
3669                         char const delimiter = p.next_token().character();
3670                         string const arg = p.getArg(delimiter, delimiter);
3671                         ostringstream oss;
3672                         oss << "\\verb" << delimiter << arg << delimiter;
3673                         handle_ert(os, oss.str(), context);
3674                 }
3675
3676                 // Problem: \= creates a tabstop inside the tabbing environment
3677                 // and else an accent. In the latter case we really would want
3678                 // \={o} instead of \= o.
3679                 else if (t.cs() == "=" && (flags & FLAG_TABBING))
3680                         handle_ert(os, t.asInput(), context);
3681
3682                 // accents (see Table 6 in Comprehensive LaTeX Symbol List)
3683                 else if (t.cs().size() == 1
3684                          && contains("\"'.=^`bcdHkrtuv~", t.cs())) {
3685                         context.check_layout(os);
3686                         // try to see whether the string is in unicodesymbols
3687                         bool termination;
3688                         docstring rem;
3689                         string command = t.asInput() + "{"
3690                                 + trimSpaceAndEol(p.verbatim_item())
3691                                 + "}";
3692                         set<string> req;
3693                         docstring s = encodings.fromLaTeXCommand(from_utf8(command),
3694                                 Encodings::TEXT_CMD | Encodings::MATH_CMD,
3695                                 termination, rem, &req);
3696                         if (!s.empty()) {
3697                                 if (!rem.empty())
3698                                         cerr << "When parsing " << command
3699                                              << ", result is " << to_utf8(s)
3700                                              << "+" << to_utf8(rem) << endl;
3701                                 os << to_utf8(s);
3702                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
3703                                         preamble.registerAutomaticallyLoadedPackage(*it);
3704                         } else
3705                                 // we did not find a non-ert version
3706                                 handle_ert(os, command, context);
3707                 }
3708
3709                 else if (t.cs() == "\\") {
3710                         context.check_layout(os);
3711                         if (p.hasOpt())
3712                                 handle_ert(os, "\\\\" + p.getOpt(), context);
3713                         else if (p.next_token().asInput() == "*") {
3714                                 p.get_token();
3715                                 // getOpt() eats the following space if there
3716                                 // is no optional argument, but that is OK
3717                                 // here since it has no effect in the output.
3718                                 handle_ert(os, "\\\\*" + p.getOpt(), context);
3719                         }
3720                         else {
3721                                 begin_inset(os, "Newline newline");
3722                                 end_inset(os);
3723                         }
3724                 }
3725
3726                 else if (t.cs() == "newline" ||
3727                          (t.cs() == "linebreak" && !p.hasOpt())) {
3728                         context.check_layout(os);
3729                         begin_inset(os, "Newline ");
3730                         os << t.cs();
3731                         end_inset(os);
3732                         skip_spaces_braces(p);
3733                 }
3734
3735                 else if (t.cs() == "input" || t.cs() == "include"
3736                          || t.cs() == "verbatiminput") {
3737                         string name = t.cs();
3738                         if (t.cs() == "verbatiminput"
3739                             && p.next_token().asInput() == "*")
3740                                 name += p.get_token().asInput();
3741                         context.check_layout(os);
3742                         string filename(normalize_filename(p.getArg('{', '}')));
3743                         string const path = getMasterFilePath();
3744                         // We want to preserve relative / absolute filenames,
3745                         // therefore path is only used for testing
3746                         if ((t.cs() == "include" || t.cs() == "input") &&
3747                             !makeAbsPath(filename, path).exists()) {
3748                                 // The file extension is probably missing.
3749                                 // Now try to find it out.
3750                                 string const tex_name =
3751                                         find_file(filename, path,
3752                                                   known_tex_extensions);
3753                                 if (!tex_name.empty())
3754                                         filename = tex_name;
3755                         }
3756                         bool external = false;
3757                         string outname;
3758                         if (makeAbsPath(filename, path).exists()) {
3759                                 string const abstexname =
3760                                         makeAbsPath(filename, path).absFileName();
3761                                 string const abslyxname =
3762                                         changeExtension(abstexname, ".lyx");
3763                                 string const absfigname =
3764                                         changeExtension(abstexname, ".fig");
3765                                 fix_relative_filename(filename);
3766                                 string const lyxname =
3767                                         changeExtension(filename, ".lyx");
3768                                 bool xfig = false;
3769                                 external = FileName(absfigname).exists();
3770                                 if (t.cs() == "input") {
3771                                         string const ext = getExtension(abstexname);
3772
3773                                         // Combined PS/LaTeX:
3774                                         // x.eps, x.pstex_t (old xfig)
3775                                         // x.pstex, x.pstex_t (new xfig, e.g. 3.2.5)
3776                                         FileName const absepsname(
3777                                                 changeExtension(abstexname, ".eps"));
3778                                         FileName const abspstexname(
3779                                                 changeExtension(abstexname, ".pstex"));
3780                                         bool const xfigeps =
3781                                                 (absepsname.exists() ||
3782                                                  abspstexname.exists()) &&
3783                                                 ext == "pstex_t";
3784
3785                                         // Combined PDF/LaTeX:
3786                                         // x.pdf, x.pdftex_t (old xfig)
3787                                         // x.pdf, x.pdf_t (new xfig, e.g. 3.2.5)
3788                                         FileName const abspdfname(
3789                                                 changeExtension(abstexname, ".pdf"));
3790                                         bool const xfigpdf =
3791                                                 abspdfname.exists() &&
3792                                                 (ext == "pdftex_t" || ext == "pdf_t");
3793                                         if (xfigpdf)
3794                                                 pdflatex = true;
3795
3796                                         // Combined PS/PDF/LaTeX:
3797                                         // x_pspdftex.eps, x_pspdftex.pdf, x.pspdftex
3798                                         string const absbase2(
3799                                                 removeExtension(abstexname) + "_pspdftex");
3800                                         FileName const abseps2name(
3801                                                 addExtension(absbase2, ".eps"));
3802                                         FileName const abspdf2name(
3803                                                 addExtension(absbase2, ".pdf"));
3804                                         bool const xfigboth =
3805                                                 abspdf2name.exists() &&
3806                                                 abseps2name.exists() && ext == "pspdftex";
3807
3808                                         xfig = xfigpdf || xfigeps || xfigboth;
3809                                         external = external && xfig;
3810                                 }
3811                                 if (external) {
3812                                         outname = changeExtension(filename, ".fig");
3813                                 } else if (xfig) {
3814                                         // Don't try to convert, the result
3815                                         // would be full of ERT.
3816                                         outname = filename;
3817                                 } else if (t.cs() != "verbatiminput" &&
3818                                     tex2lyx(abstexname, FileName(abslyxname),
3819                                             p.getEncoding())) {
3820                                         outname = lyxname;
3821                                 } else {
3822                                         outname = filename;
3823                                 }
3824                         } else {
3825                                 cerr << "Warning: Could not find included file '"
3826                                      << filename << "'." << endl;
3827                                 outname = filename;
3828                         }
3829                         if (external) {
3830                                 begin_inset(os, "External\n");
3831                                 os << "\ttemplate XFig\n"
3832                                    << "\tfilename " << outname << '\n';
3833                                 registerExternalTemplatePackages("XFig");
3834                         } else {
3835                                 begin_command_inset(os, "include", name);
3836                                 os << "preview false\n"
3837                                       "filename \"" << outname << "\"\n";
3838                                 if (t.cs() == "verbatiminput")
3839                                         preamble.registerAutomaticallyLoadedPackage("verbatim");
3840                         }
3841                         end_inset(os);
3842                 }
3843
3844                 else if (t.cs() == "bibliographystyle") {
3845                         // store new bibliographystyle
3846                         bibliographystyle = p.verbatim_item();
3847                         // If any other command than \bibliography and
3848                         // \nocite{*} follows, we need to output the style
3849                         // (because it might be used by that command).
3850                         // Otherwise, it will automatically be output by LyX.
3851                         p.pushPosition();
3852                         bool output = true;
3853                         for (Token t2 = p.get_token(); p.good(); t2 = p.get_token()) {
3854                                 if (t2.cat() == catBegin)
3855                                         break;
3856                                 if (t2.cat() != catEscape)
3857                                         continue;
3858                                 if (t2.cs() == "nocite") {
3859                                         if (p.getArg('{', '}') == "*")
3860                                                 continue;
3861                                 } else if (t2.cs() == "bibliography")
3862                                         output = false;
3863                                 break;
3864                         }
3865                         p.popPosition();
3866                         if (output) {
3867                                 handle_ert(os,
3868                                         "\\bibliographystyle{" + bibliographystyle + '}',
3869                                         context);
3870                         }
3871                 }
3872
3873                 else if (t.cs() == "bibliography") {
3874                         context.check_layout(os);
3875                         begin_command_inset(os, "bibtex", "bibtex");
3876                         if (!btprint.empty()) {
3877                                 os << "btprint " << '"' << "btPrintAll" << '"' << "\n";
3878                                 // clear the string because the next BibTeX inset can be without the
3879                                 // \nocite{*} option
3880                                 btprint.clear();
3881                         }
3882                         os << "bibfiles " << '"' << p.verbatim_item() << '"' << "\n";
3883                         // Do we have a bibliographystyle set?
3884                         if (!bibliographystyle.empty())
3885                                 os << "options " << '"' << bibliographystyle << '"' << "\n";
3886                         end_inset(os);
3887                 }
3888
3889                 else if (t.cs() == "parbox") {
3890                         // Test whether this is an outer box of a shaded box
3891                         p.pushPosition();
3892                         // swallow arguments
3893                         while (p.hasOpt()) {
3894                                 p.getArg('[', ']');
3895                                 p.skip_spaces(true);
3896                         }
3897                         p.getArg('{', '}');
3898                         p.skip_spaces(true);
3899                         // eat the '{'
3900                         if (p.next_token().cat() == catBegin)
3901                                 p.get_token();
3902                         p.skip_spaces(true);
3903                         Token to = p.get_token();
3904                         bool shaded = false;
3905                         if (to.asInput() == "\\begin") {
3906                                 p.skip_spaces(true);
3907                                 if (p.getArg('{', '}') == "shaded")
3908                                         shaded = true;
3909                         }
3910                         p.popPosition();
3911                         if (shaded) {
3912                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3913                                                 context, "parbox", "shaded");
3914                         } else
3915                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3916                                           "", "", t.cs());
3917                 }
3918
3919                 else if (t.cs() == "ovalbox" || t.cs() == "Ovalbox" ||
3920                          t.cs() == "shadowbox" || t.cs() == "doublebox")
3921                         parse_outer_box(p, os, FLAG_ITEM, outer, context, t.cs(), "");
3922
3923                 else if (t.cs() == "framebox") {
3924                         if (p.next_token().character() == '(') {
3925                                 //the syntax is: \framebox(x,y)[position]{content}
3926                                 string arg = t.asInput();
3927                                 arg += p.getFullParentheseArg();
3928                                 arg += p.getFullOpt();
3929                                 eat_whitespace(p, os, context, false);
3930                                 handle_ert(os, arg + '{', context);
3931                                 eat_whitespace(p, os, context, false);
3932                                 parse_text(p, os, FLAG_ITEM, outer, context);
3933                                 handle_ert(os, "}", context);
3934                         } else {
3935                                 string special = p.getFullOpt();
3936                                 special += p.getOpt();
3937                                 parse_outer_box(p, os, FLAG_ITEM, outer,
3938                                                 context, t.cs(), special);
3939                         }
3940                 }
3941
3942                 //\makebox() is part of the picture environment and different from \makebox{}
3943                 //\makebox{} will be parsed by parse_box
3944                 else if (t.cs() == "makebox") {
3945                         if (p.next_token().character() == '(') {
3946                                 //the syntax is: \makebox(x,y)[position]{content}
3947                                 string arg = t.asInput();
3948                                 arg += p.getFullParentheseArg();
3949                                 arg += p.getFullOpt();
3950                                 eat_whitespace(p, os, context, false);
3951                                 handle_ert(os, arg + '{', context);
3952                                 eat_whitespace(p, os, context, false);
3953                                 parse_text(p, os, FLAG_ITEM, outer, context);
3954                                 handle_ert(os, "}", context);
3955                         } else
3956                                 //the syntax is: \makebox[width][position]{content}
3957                                 parse_box(p, os, 0, FLAG_ITEM, outer, context,
3958                                           "", "", t.cs());
3959                 }
3960
3961                 else if (t.cs() == "smallskip" ||
3962                          t.cs() == "medskip" ||
3963                          t.cs() == "bigskip" ||
3964                          t.cs() == "vfill") {
3965                         context.check_layout(os);
3966                         begin_inset(os, "VSpace ");
3967                         os << t.cs();
3968                         end_inset(os);
3969                         skip_spaces_braces(p);
3970                 }
3971
3972                 else if ((where = is_known(t.cs(), known_spaces))) {
3973                         context.check_layout(os);
3974                         begin_inset(os, "space ");
3975                         os << '\\' << known_coded_spaces[where - known_spaces]
3976                            << '\n';
3977                         end_inset(os);
3978                         // LaTeX swallows whitespace after all spaces except
3979                         // "\\,". We have to do that here, too, because LyX
3980                         // adds "{}" which would make the spaces significant.
3981                         if (t.cs() !=  ",")
3982                                 eat_whitespace(p, os, context, false);
3983                         // LyX adds "{}" after all spaces except "\\ " and
3984                         // "\\,", so we have to remove "{}".
3985                         // "\\,{}" is equivalent to "\\," in LaTeX, so we
3986                         // remove the braces after "\\,", too.
3987                         if (t.cs() != " ")
3988                                 skip_braces(p);
3989                 }
3990
3991                 else if (t.cs() == "newpage" ||
3992                          (t.cs() == "pagebreak" && !p.hasOpt()) ||
3993                          t.cs() == "clearpage" ||
3994                          t.cs() == "cleardoublepage") {
3995                         context.check_layout(os);
3996                         begin_inset(os, "Newpage ");
3997                         os << t.cs();
3998                         end_inset(os);
3999                         skip_spaces_braces(p);
4000                 }
4001
4002                 else if (t.cs() == "DeclareRobustCommand" ||
4003                          t.cs() == "DeclareRobustCommandx" ||
4004                          t.cs() == "newcommand" ||
4005                          t.cs() == "newcommandx" ||
4006                          t.cs() == "providecommand" ||
4007                          t.cs() == "providecommandx" ||
4008                          t.cs() == "renewcommand" ||
4009                          t.cs() == "renewcommandx") {
4010                         // DeclareRobustCommand, DeclareRobustCommandx,
4011                         // providecommand and providecommandx could be handled
4012                         // by parse_command(), but we need to call
4013                         // add_known_command() here.
4014                         string name = t.asInput();
4015                         if (p.next_token().asInput() == "*") {
4016                                 // Starred form. Eat '*'
4017                                 p.get_token();
4018                                 name += '*';
4019                         }
4020                         string const command = p.verbatim_item();
4021                         string const opt1 = p.getFullOpt();
4022                         string const opt2 = p.getFullOpt();
4023                         add_known_command(command, opt1, !opt2.empty());
4024                         string const ert = name + '{' + command + '}' +
4025                                            opt1 + opt2 +
4026                                            '{' + p.verbatim_item() + '}';
4027
4028                         if (t.cs() == "DeclareRobustCommand" ||
4029                             t.cs() == "DeclareRobustCommandx" ||
4030                             t.cs() == "providecommand" ||
4031                             t.cs() == "providecommandx" ||
4032                             name[name.length()-1] == '*')
4033                                 handle_ert(os, ert, context);
4034                         else {
4035                                 context.check_layout(os);
4036                                 begin_inset(os, "FormulaMacro");
4037                                 os << "\n" << ert;
4038                                 end_inset(os);
4039                         }
4040                 }
4041
4042                 else if (t.cs() == "let" && p.next_token().asInput() != "*") {
4043                         // let could be handled by parse_command(),
4044                         // but we need to call add_known_command() here.
4045                         string ert = t.asInput();
4046                         string name;
4047                         p.skip_spaces();
4048                         if (p.next_token().cat() == catBegin) {
4049                                 name = p.verbatim_item();
4050                                 ert += '{' + name + '}';
4051                         } else {
4052                                 name = p.verbatim_item();
4053                                 ert += name;
4054                         }
4055                         string command;
4056                         p.skip_spaces();
4057                         if (p.next_token().cat() == catBegin) {
4058                                 command = p.verbatim_item();
4059                                 ert += '{' + command + '}';
4060                         } else {
4061                                 command = p.verbatim_item();
4062                                 ert += command;
4063                         }
4064                         // If command is known, make name known too, to parse
4065                         // its arguments correctly. For this reason we also
4066                         // have commands in syntax.default that are hardcoded.
4067                         CommandMap::iterator it = known_commands.find(command);
4068                         if (it != known_commands.end())
4069                                 known_commands[t.asInput()] = it->second;
4070                         handle_ert(os, ert, context);
4071                 }
4072
4073                 else if (t.cs() == "hspace" || t.cs() == "vspace") {
4074                         bool starred = false;
4075                         if (p.next_token().asInput() == "*") {
4076                                 p.get_token();
4077                                 starred = true;
4078                         }
4079                         string name = t.asInput();
4080                         string const length = p.verbatim_item();
4081                         string unit;
4082                         string valstring;
4083                         bool valid = splitLatexLength(length, valstring, unit);
4084                         bool known_hspace = false;
4085                         bool known_vspace = false;
4086                         bool known_unit = false;
4087                         double value;
4088                         if (valid) {
4089                                 istringstream iss(valstring);
4090                                 iss >> value;
4091                                 if (value == 1.0) {
4092                                         if (t.cs()[0] == 'h') {
4093                                                 if (unit == "\\fill") {
4094                                                         if (!starred) {
4095                                                                 unit = "";
4096                                                                 name = "\\hfill";
4097                                                         }
4098                                                         known_hspace = true;
4099                                                 }
4100                                         } else {
4101                                                 if (unit == "\\smallskipamount") {
4102                                                         unit = "smallskip";
4103                                                         known_vspace = true;
4104                                                 } else if (unit == "\\medskipamount") {
4105                                                         unit = "medskip";
4106                                                         known_vspace = true;
4107                                                 } else if (unit == "\\bigskipamount") {
4108                                                         unit = "bigskip";
4109                                                         known_vspace = true;
4110                                                 } else if (unit == "\\fill") {
4111                                                         unit = "vfill";
4112                                                         known_vspace = true;
4113                                                 }
4114                                         }
4115                                 }
4116                                 if (!known_hspace && !known_vspace) {
4117                                         switch (unitFromString(unit)) {
4118                                         case Length::SP:
4119                                         case Length::PT:
4120                                         case Length::BP:
4121                                         case Length::DD:
4122                                         case Length::MM:
4123                                         case Length::PC:
4124                                         case Length::CC:
4125                                         case Length::CM:
4126                                         case Length::IN:
4127                                         case Length::EX:
4128                                         case Length::EM:
4129                                         case Length::MU:
4130                                                 known_unit = true;
4131                                                 break;
4132                                         default:
4133                                                 break;
4134                                         }
4135                                 }
4136                         }
4137
4138                         if (t.cs()[0] == 'h' && (known_unit || known_hspace)) {
4139                                 // Literal horizontal length or known variable
4140                                 context.check_layout(os);
4141                                 begin_inset(os, "space ");
4142                                 os << name;
4143                                 if (starred)
4144                                         os << '*';
4145                                 os << '{';
4146                                 if (known_hspace)
4147                                         os << unit;
4148                                 os << "}";
4149                                 if (known_unit && !known_hspace)
4150                                         os << "\n\\length "
4151                                            << translate_len(length);
4152                                 end_inset(os);
4153                         } else if (known_unit || known_vspace) {
4154                                 // Literal vertical length or known variable
4155                                 context.check_layout(os);
4156                                 begin_inset(os, "VSpace ");
4157                                 if (known_unit)
4158                                         os << value;
4159                                 os << unit;
4160                                 if (starred)
4161                                         os << '*';
4162                                 end_inset(os);
4163                         } else {
4164                                 // LyX can't handle other length variables in Inset VSpace/space
4165                                 if (starred)
4166                                         name += '*';
4167                                 if (valid) {
4168                                         if (value == 1.0)
4169                                                 handle_ert(os, name + '{' + unit + '}', context);
4170                                         else if (value == -1.0)
4171                                                 handle_ert(os, name + "{-" + unit + '}', context);
4172                                         else
4173                                                 handle_ert(os, name + '{' + valstring + unit + '}', context);
4174                                 } else
4175                                         handle_ert(os, name + '{' + length + '}', context);
4176                         }
4177                 }
4178
4179                 // The single '=' is meant here.
4180                 else if ((newinsetlayout = findInsetLayout(context.textclass, t.cs(), true))) {
4181                         p.skip_spaces();
4182                         context.check_layout(os);
4183                         begin_inset(os, "Flex ");
4184                         os << to_utf8(newinsetlayout->name()) << '\n'
4185                            << "status collapsed\n";
4186                         parse_text_in_inset(p, os, FLAG_ITEM, false, context, newinsetlayout);
4187                         end_inset(os);
4188                 }
4189
4190                 else if (t.cs() == "includepdf") {
4191                         p.skip_spaces();
4192                         string const arg = p.getArg('[', ']');
4193                         map<string, string> opts;
4194                         vector<string> keys;
4195                         split_map(arg, opts, keys);
4196                         string name = normalize_filename(p.verbatim_item());
4197                         string const path = getMasterFilePath();
4198                         // We want to preserve relative / absolute filenames,
4199                         // therefore path is only used for testing
4200                         if (!makeAbsPath(name, path).exists()) {
4201                                 // The file extension is probably missing.
4202                                 // Now try to find it out.
4203                                 char const * const pdfpages_format[] = {"pdf", 0};
4204                                 string const pdftex_name =
4205                                         find_file(name, path, pdfpages_format);
4206                                 if (!pdftex_name.empty()) {
4207                                         name = pdftex_name;
4208                                         pdflatex = true;
4209                                 }
4210                         }
4211                         if (makeAbsPath(name, path).exists())
4212                                 fix_relative_filename(name);
4213                         else
4214                                 cerr << "Warning: Could not find file '"
4215                                      << name << "'." << endl;
4216                         // write output
4217                         context.check_layout(os);
4218                         begin_inset(os, "External\n\ttemplate ");
4219                         os << "PDFPages\n\tfilename "
4220                            << name << "\n";
4221                         // parse the options
4222                         if (opts.find("pages") != opts.end())
4223                                 os << "\textra LaTeX \"pages="
4224                                    << opts["pages"] << "\"\n";
4225                         if (opts.find("angle") != opts.end())
4226                                 os << "\trotateAngle "
4227                                    << opts["angle"] << '\n';
4228                         if (opts.find("origin") != opts.end()) {
4229                                 ostringstream ss;
4230                                 string const opt = opts["origin"];
4231                                 if (opt == "tl") ss << "topleft";
4232                                 if (opt == "bl") ss << "bottomleft";
4233                                 if (opt == "Bl") ss << "baselineleft";
4234                                 if (opt == "c") ss << "center";
4235                                 if (opt == "tc") ss << "topcenter";
4236                                 if (opt == "bc") ss << "bottomcenter";
4237                                 if (opt == "Bc") ss << "baselinecenter";
4238                                 if (opt == "tr") ss << "topright";
4239                                 if (opt == "br") ss << "bottomright";
4240                                 if (opt == "Br") ss << "baselineright";
4241                                 if (!ss.str().empty())
4242                                         os << "\trotateOrigin " << ss.str() << '\n';
4243                                 else
4244                                         cerr << "Warning: Ignoring unknown includegraphics origin argument '" << opt << "'\n";
4245                         }
4246                         if (opts.find("width") != opts.end())
4247                                 os << "\twidth "
4248                                    << translate_len(opts["width"]) << '\n';
4249                         if (opts.find("height") != opts.end())
4250                                 os << "\theight "
4251                                    << translate_len(opts["height"]) << '\n';
4252                         if (opts.find("keepaspectratio") != opts.end())
4253                                 os << "\tkeepAspectRatio\n";
4254                         end_inset(os);
4255                         context.check_layout(os);
4256                         registerExternalTemplatePackages("PDFPages");
4257                 }
4258
4259                 else if (t.cs() == "loadgame") {
4260                         p.skip_spaces();
4261                         string name = normalize_filename(p.verbatim_item());
4262                         string const path = getMasterFilePath();
4263                         // We want to preserve relative / absolute filenames,
4264                         // therefore path is only used for testing
4265                         if (!makeAbsPath(name, path).exists()) {
4266                                 // The file extension is probably missing.
4267                                 // Now try to find it out.
4268                                 char const * const lyxskak_format[] = {"fen", 0};
4269                                 string const lyxskak_name =
4270                                         find_file(name, path, lyxskak_format);
4271                                 if (!lyxskak_name.empty())
4272                                         name = lyxskak_name;
4273                         }
4274                         if (makeAbsPath(name, path).exists())
4275                                 fix_relative_filename(name);
4276                         else
4277                                 cerr << "Warning: Could not find file '"
4278                                      << name << "'." << endl;
4279                         context.check_layout(os);
4280                         begin_inset(os, "External\n\ttemplate ");
4281                         os << "ChessDiagram\n\tfilename "
4282                            << name << "\n";
4283                         end_inset(os);
4284                         context.check_layout(os);
4285                         // after a \loadgame follows a \showboard
4286                         if (p.get_token().asInput() == "showboard")
4287                                 p.get_token();
4288                         registerExternalTemplatePackages("ChessDiagram");
4289                 }
4290
4291                 else {
4292                         // try to see whether the string is in unicodesymbols
4293                         // Only use text mode commands, since we are in text mode here,
4294                         // and math commands may be invalid (bug 6797)
4295                         bool termination;
4296                         docstring rem;
4297                         set<string> req;
4298                         docstring s = encodings.fromLaTeXCommand(from_utf8(t.asInput()),
4299                                         Encodings::TEXT_CMD, termination, rem, &req);
4300                         if (!s.empty()) {
4301                                 if (!rem.empty())
4302                                         cerr << "When parsing " << t.cs()
4303                                              << ", result is " << to_utf8(s)
4304                                              << "+" << to_utf8(rem) << endl;
4305                                 context.check_layout(os);
4306                                 os << to_utf8(s);
4307                                 if (termination)
4308                                         skip_spaces_braces(p);
4309                                 for (set<string>::const_iterator it = req.begin(); it != req.end(); ++it)
4310                                         preamble.registerAutomaticallyLoadedPackage(*it);
4311                         }
4312                         //cerr << "#: " << t << " mode: " << mode << endl;
4313                         // heuristic: read up to next non-nested space
4314                         /*
4315                         string s = t.asInput();
4316                         string z = p.verbatim_item();
4317                         while (p.good() && z != " " && z.size()) {
4318                                 //cerr << "read: " << z << endl;
4319                                 s += z;
4320                                 z = p.verbatim_item();
4321                         }
4322                         cerr << "found ERT: " << s << endl;
4323                         handle_ert(os, s + ' ', context);
4324                         */
4325                         else {
4326                                 string name = t.asInput();
4327                                 if (p.next_token().asInput() == "*") {
4328                                         // Starred commands like \vspace*{}
4329                                         p.get_token();  // Eat '*'
4330                                         name += '*';
4331                                 }
4332                                 if (!parse_command(name, p, os, outer, context))
4333                                         handle_ert(os, name, context);
4334                         }
4335                 }
4336
4337                 if (flags & FLAG_LEAVE) {
4338                         flags &= ~FLAG_LEAVE;
4339                         break;
4340                 }
4341         }
4342 }
4343
4344 // }])
4345
4346
4347 } // namespace lyx